From 4f3b88db0ed5deaf5e8f77ab451e5759940af199 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 16 Oct 2026 11:06:43 +0200 Subject: [PATCH 01/50] feat: CARP addresses as virtual IPs, with their holder in the reading (4.10.0) A firewall pair's CARP addresses are a third virtual IP mechanism. Unlike vip-manager and keepalived, a CARP reading says who holds the address: a firewall's interface list need not show it, but CARP's state names the MASTER. - VirtualIpsDict gains carp (NotRequired), VirtualIpDict gains holds (NotRequired; None leaves the holder to the interfaces). - carp_virtual_ip() shapes one address bound to a VHID: MASTER holds, BACKUP stands by, INIT and DISABLED neither run nor hold, an unknown state says nothing. The same for OPNsense and pfSense; a driver supplies the fields. netOrk #1121. --- README.md | 8 +++ napalm_device_types/__init__.py | 2 + napalm_device_types/models.py | 29 +++++--- napalm_device_types/virtual_ips.py | 52 ++++++++++++++ pyproject.toml | 2 +- tests/test_carp_virtual_ips.py | 111 +++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 tests/test_carp_virtual_ips.py diff --git a/README.md b/README.md index 1f90cef..542f9df 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,14 @@ A mechanism it could not read completely is `None` ("did not look"), a host with `[]`. The command is about 4 kB, so a driver sends it on an exec channel (`_run_virtual_ips_command(command, privileged=)`), not typed into a shell. +A firewall's CARP addresses are a third mechanism, `carp`, which a firewall driver reads +from its own API and reports under the reading's `carp` key (`vip_manager` and +`keepalived` then None). `carp_virtual_ip(...)` turns one address bound to a VHID into the +entry -- the same for OPNsense and pfSense. Unlike the other two, a CARP entry says who +holds the address (`holds`): every member carries the address in its configuration, and a +firewall's interface list need not show it, but CARP's state names the MASTER. A state it +does not know leaves `holds` None, and the caller falls back on the interfaces. + `SystemIdentityMixin` (`get_system_identity`) reads a host's SMBIOS identity -- system UUID, product serial and board serial from `/sys/class/dmi/id` -- which is what ties a server to its BMC (`ManagementControllerDriver.get_hardware_inventory` reports the same values from diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 5eb32f9..4d901a8 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -134,6 +134,7 @@ from napalm_device_types.system_identity import ( from napalm_device_types.virtual_ips import ( VIRTUAL_IPS_COMMAND, VirtualIpsMixin, + carp_virtual_ip, parse_virtual_ips, ) from napalm_device_types.residential_gateway import ResidentialGatewayDriver @@ -221,6 +222,7 @@ __all__ = [ "SshHardeningResult", "plan_ssh_hardening", "parse_virtual_ips", + "carp_virtual_ip", "add_lag_interfaces", "driver_supports_ping", "normalize_cidr", diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index cc5e41c..f730be1 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -404,29 +404,42 @@ class VirtualIpDict(TypedDict): """An address the host's HA configuration lets float between machines. Declared, not observed: whether the host holds the address right now is in - its interface addresses, not here. Only these fields leave the host -- never - a password, an etcd endpoint or a notify script. + its interface addresses, not here -- unless the mechanism says so itself. + CARP does (``holds``): a firewall's interface list need not show the + address at all, but CARP's state names the MASTER. Only these fields leave + the host -- never a password, an etcd endpoint or a notify script. """ - mechanism: str # "vip-manager" | "keepalived" - instance: str # the vrrp_instance name, or the vip-manager unit without ".service" + mechanism: str # "vip-manager" | "keepalived" | "carp" + #: the vrrp_instance name, the vip-manager unit without ".service", or the + #: CARP address's interface description + instance: str address: str # normalised: "10.7.224.10", "fd00::10" prefix_length: Optional[int] # None where the configuration gives none interface: Optional[str] - group_key: Optional[str] # vip-manager's trigger-key, or "vrid:" for keepalived - running: Optional[bool] # the unit is active + #: vip-manager's trigger-key, "vrid:" for keepalived, "vhid:" for CARP + group_key: Optional[str] + running: Optional[bool] # the unit is active; for CARP: in MASTER or BACKUP + #: The host holds the address now, by the mechanism's own state (CARP's + #: MASTER). Absent or None: the interface addresses tell. + holds: NotRequired[Optional[bool]] class VirtualIpsDict(TypedDict): - """What ``VirtualIpsMixin.get_virtual_ips`` read, per mechanism. + """What a driver's ``get_virtual_ips`` read, per mechanism. A list is what was looked at -- ``[]`` a host without that mechanism. None is "did not look": a configuration it could not read, an include it could not - follow, a host without systemd. A consumer keeps what it knew for a None. + follow, a host without systemd, a mechanism the driver does not read (a + firewall's reading has ``vip_manager`` and ``keepalived`` None). A consumer + keeps what it knew for a None, and for a missing ``carp``. """ vip_manager: Optional[List[VirtualIpDict]] keepalived: Optional[List[VirtualIpDict]] + #: A firewall's CARP addresses (not read by ``VirtualIpsMixin``, which reads Linux + #: hosts): every address bound to a VHID, the CARP address and IP aliases on it. + carp: NotRequired[Optional[List[VirtualIpDict]]] class PortForwardDict(TypedDict): diff --git a/napalm_device_types/virtual_ips.py b/napalm_device_types/virtual_ips.py index 6733684..565243f 100644 --- a/napalm_device_types/virtual_ips.py +++ b/napalm_device_types/virtual_ips.py @@ -42,6 +42,11 @@ caller keeps what it knew. ``[]`` is a host that has none. script goes to the host as one ``sh -c`` argument; when the privileged call brings no report back, it runs again without privilege and reads what it can. +**CARP.** A firewall's CARP addresses come from its own API, not from this +command, so :func:`carp_virtual_ip` only shapes one entry: what is the same for +every CARP firewall is the meaning of the state. CARP's MASTER holds the address +(``holds``), which the interface list of a firewall need not show. + **An exec channel, not a terminal.** With its awk program the command is about 4 kB, past the line an interactive shell reliably takes. A driver sends it the way it runs any command that needs no PTY -- napalm-linux's ``run_command``. @@ -332,6 +337,53 @@ _READERS: Dict[str, Tuple[str, Callable[[str, List[_Record]], List[VirtualIpDict } +#: CARP's state of a VHID on one member -> (running, holds). +_CARP_STATES: Dict[str, Tuple[bool, bool]] = { + "MASTER": (True, True), + "BACKUP": (True, False), + "INIT": (False, False), + "DISABLED": (False, False), +} + + +def carp_virtual_ip( + *, + address: str, + prefix_length: Optional[int], + vhid: object, + state: Optional[str], + instance: str, + interface: Optional[str], +) -> Optional[VirtualIpDict]: + """One address bound to a CARP VHID, as a virtual IP; None for what is no address. + + Who holds it comes from CARP's *state* on this member, not from its + interfaces: MASTER holds it, BACKUP stands by, INIT and DISABLED neither run + nor hold. A state this does not know says nothing (``running`` and ``holds`` + None), and the caller falls back on the interfaces. *address* may carry its + prefix (``"10.0.0.1/24"``), which then wins over *prefix_length*. + """ + parsed = _address(address.strip()) if address else None + if parsed is None: + return None + normalised, written = parsed + prefix = written if written is not None else prefix_length + if prefix is not None and not 0 <= prefix <= ipaddress.ip_address(normalised).max_prefixlen: + prefix = None + vhid_text = str(vhid).strip() if vhid is not None else "" + running, holds = _CARP_STATES.get((state or "").strip().upper(), (None, None)) + return { + "mechanism": "carp", + "instance": instance or f"vhid {vhid_text}", + "address": normalised, + "prefix_length": prefix, + "interface": interface, + "group_key": f"vhid:{vhid_text}" if vhid_text.isdigit() else None, + "running": running, + "holds": holds, + } + + def _order(entry: VirtualIpDict) -> Tuple[int, int, str]: address = ipaddress.ip_address(entry["address"]) return address.version, int(address), entry["instance"] diff --git a/pyproject.toml b/pyproject.toml index 2b541c9..b965bed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.9.0" +version = "4.10.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_carp_virtual_ips.py b/tests/test_carp_virtual_ips.py new file mode 100644 index 0000000..3568a5e --- /dev/null +++ b/tests/test_carp_virtual_ips.py @@ -0,0 +1,111 @@ +"""carp_virtual_ip: one CARP address as a virtual IP, with its holder in the reading. + +CARP differs from vip-manager and keepalived in what the caller can see. The +address is configured on every member, and the interface list of a firewall need +not show it at all -- so the interfaces cannot say who holds it. CARP's own state +can: the member in MASTER holds it, a BACKUP stands by. Turning that state into a +virtual IP is the same for every CARP firewall (OPNsense, pfSense), so it lives +here; a driver supplies the fields from its API (netOrk #1121). +""" + +from __future__ import annotations + +import pytest + +from napalm_device_types.virtual_ips import carp_virtual_ip + + +def _entry(**overrides): + fields = { + "address": "172.22.8.1", + "prefix_length": 24, + "vhid": "108", + "state": "MASTER", + "instance": "MGMT", + "interface": "vtnet0", + } + fields.update(overrides) + return carp_virtual_ip(**fields) + + +def test_a_master_holds_the_address(): + assert _entry() == { + "mechanism": "carp", + "instance": "MGMT", + "address": "172.22.8.1", + "prefix_length": 24, + "interface": "vtnet0", + "group_key": "vhid:108", + "running": True, + "holds": True, + } + + +def test_a_backup_stands_by(): + entry = _entry(state="BACKUP") + assert entry is not None + assert (entry["running"], entry["holds"]) == (True, False) + + +@pytest.mark.parametrize("state", ["INIT", "DISABLED"]) +def test_a_member_whose_carp_is_not_up_neither_runs_nor_holds(state): + entry = _entry(state=state) + assert entry is not None + assert (entry["running"], entry["holds"]) == (False, False) + + +@pytest.mark.parametrize("state", [None, "", "SOMETHING-NEW"]) +def test_an_unknown_state_says_nothing_about_the_holder(state): + """None leaves the holder to the interfaces, as for the other mechanisms.""" + entry = _entry(state=state) + assert entry is not None + assert (entry["running"], entry["holds"]) == (None, None) + + +def test_the_state_is_read_case_blind(): + entry = _entry(state=" master ") + assert entry is not None + assert entry["holds"] is True + + +def test_the_address_is_normalised(): + entry = _entry(address="FD00::0001", prefix_length=64) + assert entry is not None + assert entry["address"] == "fd00::1" + + +def test_an_address_written_with_its_prefix_takes_that_prefix(): + entry = _entry(address="10.42.100.1/24", prefix_length=None) + assert entry is not None + assert (entry["address"], entry["prefix_length"]) == ("10.42.100.1", 24) + + +@pytest.mark.parametrize("address", ["", "not-an-address", "10.0.0.300"]) +def test_what_is_no_address_is_no_virtual_ip(address): + assert _entry(address=address) is None + + +@pytest.mark.parametrize("prefix", [33, -1]) +def test_an_impossible_prefix_is_dropped(prefix): + entry = _entry(prefix_length=prefix) + assert entry is not None + assert entry["prefix_length"] is None + + +@pytest.mark.parametrize("vhid", [None, "", "abc"]) +def test_without_a_numeric_vhid_there_is_no_group(vhid): + entry = _entry(vhid=vhid) + assert entry is not None + assert entry["group_key"] is None + + +def test_a_vhid_given_as_a_number_groups_the_same_way(): + entry = _entry(vhid=108) + assert entry is not None + assert entry["group_key"] == "vhid:108" + + +def test_an_empty_instance_falls_back_to_the_vhid(): + entry = _entry(instance="") + assert entry is not None + assert entry["instance"] == "vhid 108" From 0330f17a672fbfd8ed2638b416decec3036dec00 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 16 Oct 2026 11:11:16 +0200 Subject: [PATCH 02/50] fix: keep CARP state knowledge out of the base package (4.10.1) carp_virtual_ip() and its state table are CARP protocol knowledge; under the agreed structure (PR #37, docs/STRUCTURE.md) the base package carries only contracts, data shapes and I/O-free reconcilers. They move to the driver that speaks CARP (napalm-opnsense). The contract stays: VirtualIpsDict.carp and VirtualIpDict.holds. Nothing used the helper yet; it was released in 4.10.0 an hour ago. netOrk #1121. --- README.md | 13 ++-- napalm_device_types/__init__.py | 2 - napalm_device_types/virtual_ips.py | 52 -------------- pyproject.toml | 2 +- tests/test_carp_virtual_ips.py | 111 ----------------------------- 5 files changed, 7 insertions(+), 173 deletions(-) delete mode 100644 tests/test_carp_virtual_ips.py diff --git a/README.md b/README.md index 542f9df..a5c42d7 100644 --- a/README.md +++ b/README.md @@ -128,13 +128,12 @@ A mechanism it could not read completely is `None` ("did not look"), a host with `[]`. The command is about 4 kB, so a driver sends it on an exec channel (`_run_virtual_ips_command(command, privileged=)`), not typed into a shell. -A firewall's CARP addresses are a third mechanism, `carp`, which a firewall driver reads -from its own API and reports under the reading's `carp` key (`vip_manager` and -`keepalived` then None). `carp_virtual_ip(...)` turns one address bound to a VHID into the -entry -- the same for OPNsense and pfSense. Unlike the other two, a CARP entry says who -holds the address (`holds`): every member carries the address in its configuration, and a -firewall's interface list need not show it, but CARP's state names the MASTER. A state it -does not know leaves `holds` None, and the caller falls back on the interfaces. +A firewall's CARP addresses are a third mechanism, `carp`: a firewall driver reads them from +its own API and reports them under the reading's `carp` key (`vip_manager` and `keepalived` +then None). What a CARP state means is protocol knowledge and stays in that driver. The +contract only carries the result: unlike the other two, a CARP entry says who holds the +address (`holds`), because a firewall's interface list need not show it. `holds` None +leaves the holder to the interfaces. `SystemIdentityMixin` (`get_system_identity`) reads a host's SMBIOS identity -- system UUID, product serial and board serial from `/sys/class/dmi/id` -- which is what ties a server to diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 4d901a8..5eb32f9 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -134,7 +134,6 @@ from napalm_device_types.system_identity import ( from napalm_device_types.virtual_ips import ( VIRTUAL_IPS_COMMAND, VirtualIpsMixin, - carp_virtual_ip, parse_virtual_ips, ) from napalm_device_types.residential_gateway import ResidentialGatewayDriver @@ -222,7 +221,6 @@ __all__ = [ "SshHardeningResult", "plan_ssh_hardening", "parse_virtual_ips", - "carp_virtual_ip", "add_lag_interfaces", "driver_supports_ping", "normalize_cidr", diff --git a/napalm_device_types/virtual_ips.py b/napalm_device_types/virtual_ips.py index 565243f..6733684 100644 --- a/napalm_device_types/virtual_ips.py +++ b/napalm_device_types/virtual_ips.py @@ -42,11 +42,6 @@ caller keeps what it knew. ``[]`` is a host that has none. script goes to the host as one ``sh -c`` argument; when the privileged call brings no report back, it runs again without privilege and reads what it can. -**CARP.** A firewall's CARP addresses come from its own API, not from this -command, so :func:`carp_virtual_ip` only shapes one entry: what is the same for -every CARP firewall is the meaning of the state. CARP's MASTER holds the address -(``holds``), which the interface list of a firewall need not show. - **An exec channel, not a terminal.** With its awk program the command is about 4 kB, past the line an interactive shell reliably takes. A driver sends it the way it runs any command that needs no PTY -- napalm-linux's ``run_command``. @@ -337,53 +332,6 @@ _READERS: Dict[str, Tuple[str, Callable[[str, List[_Record]], List[VirtualIpDict } -#: CARP's state of a VHID on one member -> (running, holds). -_CARP_STATES: Dict[str, Tuple[bool, bool]] = { - "MASTER": (True, True), - "BACKUP": (True, False), - "INIT": (False, False), - "DISABLED": (False, False), -} - - -def carp_virtual_ip( - *, - address: str, - prefix_length: Optional[int], - vhid: object, - state: Optional[str], - instance: str, - interface: Optional[str], -) -> Optional[VirtualIpDict]: - """One address bound to a CARP VHID, as a virtual IP; None for what is no address. - - Who holds it comes from CARP's *state* on this member, not from its - interfaces: MASTER holds it, BACKUP stands by, INIT and DISABLED neither run - nor hold. A state this does not know says nothing (``running`` and ``holds`` - None), and the caller falls back on the interfaces. *address* may carry its - prefix (``"10.0.0.1/24"``), which then wins over *prefix_length*. - """ - parsed = _address(address.strip()) if address else None - if parsed is None: - return None - normalised, written = parsed - prefix = written if written is not None else prefix_length - if prefix is not None and not 0 <= prefix <= ipaddress.ip_address(normalised).max_prefixlen: - prefix = None - vhid_text = str(vhid).strip() if vhid is not None else "" - running, holds = _CARP_STATES.get((state or "").strip().upper(), (None, None)) - return { - "mechanism": "carp", - "instance": instance or f"vhid {vhid_text}", - "address": normalised, - "prefix_length": prefix, - "interface": interface, - "group_key": f"vhid:{vhid_text}" if vhid_text.isdigit() else None, - "running": running, - "holds": holds, - } - - def _order(entry: VirtualIpDict) -> Tuple[int, int, str]: address = ipaddress.ip_address(entry["address"]) return address.version, int(address), entry["instance"] diff --git a/pyproject.toml b/pyproject.toml index b965bed..7dec00f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.10.0" +version = "4.10.1" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_carp_virtual_ips.py b/tests/test_carp_virtual_ips.py deleted file mode 100644 index 3568a5e..0000000 --- a/tests/test_carp_virtual_ips.py +++ /dev/null @@ -1,111 +0,0 @@ -"""carp_virtual_ip: one CARP address as a virtual IP, with its holder in the reading. - -CARP differs from vip-manager and keepalived in what the caller can see. The -address is configured on every member, and the interface list of a firewall need -not show it at all -- so the interfaces cannot say who holds it. CARP's own state -can: the member in MASTER holds it, a BACKUP stands by. Turning that state into a -virtual IP is the same for every CARP firewall (OPNsense, pfSense), so it lives -here; a driver supplies the fields from its API (netOrk #1121). -""" - -from __future__ import annotations - -import pytest - -from napalm_device_types.virtual_ips import carp_virtual_ip - - -def _entry(**overrides): - fields = { - "address": "172.22.8.1", - "prefix_length": 24, - "vhid": "108", - "state": "MASTER", - "instance": "MGMT", - "interface": "vtnet0", - } - fields.update(overrides) - return carp_virtual_ip(**fields) - - -def test_a_master_holds_the_address(): - assert _entry() == { - "mechanism": "carp", - "instance": "MGMT", - "address": "172.22.8.1", - "prefix_length": 24, - "interface": "vtnet0", - "group_key": "vhid:108", - "running": True, - "holds": True, - } - - -def test_a_backup_stands_by(): - entry = _entry(state="BACKUP") - assert entry is not None - assert (entry["running"], entry["holds"]) == (True, False) - - -@pytest.mark.parametrize("state", ["INIT", "DISABLED"]) -def test_a_member_whose_carp_is_not_up_neither_runs_nor_holds(state): - entry = _entry(state=state) - assert entry is not None - assert (entry["running"], entry["holds"]) == (False, False) - - -@pytest.mark.parametrize("state", [None, "", "SOMETHING-NEW"]) -def test_an_unknown_state_says_nothing_about_the_holder(state): - """None leaves the holder to the interfaces, as for the other mechanisms.""" - entry = _entry(state=state) - assert entry is not None - assert (entry["running"], entry["holds"]) == (None, None) - - -def test_the_state_is_read_case_blind(): - entry = _entry(state=" master ") - assert entry is not None - assert entry["holds"] is True - - -def test_the_address_is_normalised(): - entry = _entry(address="FD00::0001", prefix_length=64) - assert entry is not None - assert entry["address"] == "fd00::1" - - -def test_an_address_written_with_its_prefix_takes_that_prefix(): - entry = _entry(address="10.42.100.1/24", prefix_length=None) - assert entry is not None - assert (entry["address"], entry["prefix_length"]) == ("10.42.100.1", 24) - - -@pytest.mark.parametrize("address", ["", "not-an-address", "10.0.0.300"]) -def test_what_is_no_address_is_no_virtual_ip(address): - assert _entry(address=address) is None - - -@pytest.mark.parametrize("prefix", [33, -1]) -def test_an_impossible_prefix_is_dropped(prefix): - entry = _entry(prefix_length=prefix) - assert entry is not None - assert entry["prefix_length"] is None - - -@pytest.mark.parametrize("vhid", [None, "", "abc"]) -def test_without_a_numeric_vhid_there_is_no_group(vhid): - entry = _entry(vhid=vhid) - assert entry is not None - assert entry["group_key"] is None - - -def test_a_vhid_given_as_a_number_groups_the_same_way(): - entry = _entry(vhid=108) - assert entry is not None - assert entry["group_key"] == "vhid:108" - - -def test_an_empty_instance_falls_back_to_the_vhid(): - entry = _entry(instance="") - assert entry is not None - assert entry["instance"] == "vhid 108" From 90367c13703d661ef3d4cb71a8a16a908a199acc Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 16 Oct 2026 15:18:25 +0200 Subject: [PATCH 03/50] feat: an HA pair's DHCP peers and config sync as facts (4.11.0) get_dhcp_ha (declared on DhcpServerMixin, optional) is what a DHCP server's own HA configuration says: on or off, this server's peer name, each peer's name, role as the server words it, and HA URL. ConfigSyncMixin.get_config_sync is where a host pushes its configuration and for which services, by the names get_services reports; sync items without a service name stay visible under `unmapped`. A driver mixes it in, no role promises it. Both raise when they could not read. What the facts mean for a pair -- hot standby, which side to write to -- is the caller's (NetOrk/netork#1125). --- README.md | 11 +++++ napalm_device_types/__init__.py | 9 ++++ napalm_device_types/config_sync.py | 35 +++++++++++++++ napalm_device_types/dhcp.py | 15 +++++++ napalm_device_types/models.py | 42 ++++++++++++++++++ pyproject.toml | 2 +- tests/test_dhcp_ha_config_sync.py | 70 ++++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/config_sync.py create mode 100644 tests/test_dhcp_ha_config_sync.py diff --git a/README.md b/README.md index a5c42d7..817a8fa 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,17 @@ A mechanism it could not read completely is `None` ("did not look"), a host with `[]`. The command is about 4 kB, so a driver sends it on an exec channel (`_run_virtual_ips_command(command, privileged=)`), not typed into a shell. +`get_dhcp_ha` (declared on `DhcpServerMixin`, optional) is what a DHCP server's own HA +configuration says: whether HA is on, which peer this server is, each peer's name, role -- +as the server words it, not a shared vocabulary -- and, where the server has one, the URL +its HA listener answers on. It is configuration, not state: whether the pair is in hot +standby right now is not in it. `ConfigSyncMixin` (`get_config_sync`) is where a host +pushes its configuration (`peer`, None for no one) and for which services, by the names +`get_services` reports; sync items the driver has no service name for stay visible under +`unmapped`. A driver mixes it in -- no role promises it. Both getters raise when they could +not read; neither answers "off" for "could not ask". What the facts mean for a pair (hot +standby, which side to write to) is the caller's to work out. + A firewall's CARP addresses are a third mechanism, `carp`: a firewall driver reads them from its own API and reports them under the reading's `carp` key (`vip_manager` and `keepalived` then None). What a CARP state means is protocol knowledge and stays in that driver. The diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 5eb32f9..bad9703 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -37,6 +37,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.channel.CommandChannelMixin` * :class:`~napalm_device_types.config_lifecycle.ConfigLifecycleMixin` +* :class:`~napalm_device_types.config_sync.ConfigSyncMixin` * :class:`~napalm_device_types.container_engine.ContainerEngineMixin` * :class:`~napalm_device_types.dhcp.DhcpServerMixin` * :class:`~napalm_device_types.firewall_rules.FirewallRuleMixin` @@ -71,6 +72,7 @@ from napalm_device_types.channel import ( run_on_transport, ) from napalm_device_types.config_lifecycle import ConfigLifecycleMixin +from napalm_device_types.config_sync import ConfigSyncMixin from napalm_device_types.container_engine import ( ContainerEngineConnection, ContainerEngineMixin, @@ -97,6 +99,9 @@ from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver from napalm_device_types.models import ( POWER_ACTIONS, + ConfigSyncDict, + DhcpHaDict, + DhcpHaPeerDict, EventLogEntryDict, FirmwareDict, HardwareInventoryDict, @@ -159,6 +164,10 @@ __all__ = [ "open_stream_on_transport", "run_on_transport", "ConfigLifecycleMixin", + "ConfigSyncDict", + "ConfigSyncMixin", + "DhcpHaDict", + "DhcpHaPeerDict", "DeviceTypeDriver", "DhcpServerMixin", "FingerprintRule", diff --git a/napalm_device_types/config_sync.py b/napalm_device_types/config_sync.py new file mode 100644 index 0000000..8238ca5 --- /dev/null +++ b/napalm_device_types/config_sync.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +"""Configuration sync between the two hosts of an HA pair. + +A host that pushes its configuration to a peer (OPNsense's XMLRPC sync, say) +makes the peer's configuration of the synced services a copy of its own: a +change written on the receiving side lasts only until the next push. Who pushes +what is therefore a fact a caller needs before it writes to either host. + +Declared under ``if TYPE_CHECKING``: a contract, not a placeholder. A driver +mixes it in; no role promises it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import ConfigSyncDict + + +class ConfigSyncMixin: + """Where a host pushes its configuration, and for which of its services.""" + + if TYPE_CHECKING: + + def get_config_sync(self) -> ConfigSyncDict: + """ + Returns the peer this host pushes its configuration to (None when it + pushes to no one) and the services whose configuration it pushes, by + the name ``get_services`` reports. Sync items the driver has no + service name for are listed under ``unmapped`` as the host names them. + + :raises Exception: When the setting could not be read -- never + ``peer=None`` for "could not ask". + """ + ... diff --git a/napalm_device_types/dhcp.py b/napalm_device_types/dhcp.py index 0992c63..281157e 100644 --- a/napalm_device_types/dhcp.py +++ b/napalm_device_types/dhcp.py @@ -28,6 +28,7 @@ from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( + DhcpHaDict, DhcpReservationDiffDict, DhcpReservationDict, DhcpReservationUpdateDict, @@ -219,6 +220,20 @@ class DhcpServerMixin: """ ... + def get_dhcp_ha(self) -> DhcpHaDict: + """ + Returns what the server's own HA configuration declares: whether HA + is switched on, which peer this server is, and each peer with its + role as the server words it. + + Optional: a driver without it has no ``get_dhcp_ha`` at all, and a + caller asks ``callable(getattr(driver, "get_dhcp_ha", None))``. + + :raises Exception: When the configuration could not be read -- never + ``enabled=False`` for "could not ask". + """ + ... + # ------------------------------------------------------------------ # Generic, vendor-neutral algorithms. # ------------------------------------------------------------------ diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index f730be1..5dff889 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -600,6 +600,48 @@ class DhcpSubnetDiffDict(TypedDict): update: List[DhcpSubnetUpdateDict] +class DhcpHaPeerDict(TypedDict): + """One server of a DHCP HA relationship, as the server's configuration names it.""" + + name: str + #: The peer's role as the server words it ("primary", "standby", ...) -- data, + #: not a vocabulary every server shares. + role: str + #: Where the peer's HA listener answers, when the server addresses it by URL. + url: NotRequired[str] + + +class DhcpHaDict(TypedDict): + """What a DHCP server's own HA configuration declares. + + Configured, not observed: whether the pair is in hot standby right now, or + one side has taken over, is the server's state and not in here. + """ + + #: The name of the service this configuration belongs to, as ``get_services`` + #: reports it -- the caller matches it to that service. + service: NotRequired[str] + enabled: bool + #: Which of ``peers`` is this server. + this_server: NotRequired[str] + peers: List[DhcpHaPeerDict] + + +class ConfigSyncDict(TypedDict): + """Where a host pushes its configuration, and whose. + + ``peer`` None: the host pushes to no one (the receiving side of a pair, or a + host without the feature switched on). + """ + + peer: Optional[str] + #: The services whose configuration it pushes, by the name ``get_services`` + #: reports. + services: List[str] + #: Sync items the driver has no service name for, as the host names them. + unmapped: List[str] + + class VPNTunnelDict(TypedDict): type: str local_endpoint: str diff --git a/pyproject.toml b/pyproject.toml index 7dec00f..4cc9830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.10.1" +version = "4.11.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_dhcp_ha_config_sync.py b/tests/test_dhcp_ha_config_sync.py new file mode 100644 index 0000000..89ccefa --- /dev/null +++ b/tests/test_dhcp_ha_config_sync.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +"""Two facts an HA pair of servers reports about itself. + +`get_dhcp_ha` is what a DHCP server's own HA configuration says: the peers it +pairs with and each peer's role, as the server words it. `get_config_sync` is +where a host pushes its configuration and for which of its services. Both are +declarations; what they mean for the pair -- hot standby, who to write to -- +is the caller's to work out. +""" + +from __future__ import annotations + +import inspect + +import napalm_device_types as package +from napalm_device_types import ConfigSyncMixin, DhcpServerMixin, FirewallDriver + + +class TestDeclaredOnly: + def test_get_dhcp_ha_is_absent_until_a_driver_implements_it(self): + assert not hasattr(DhcpServerMixin, "get_dhcp_ha") + + def test_get_config_sync_is_absent_until_a_driver_implements_it(self): + assert not hasattr(ConfigSyncMixin, "get_config_sync") + + def test_config_sync_mixin_owns_nothing_callable(self): + own = [ + name + for name, value in vars(ConfigSyncMixin).items() + if not name.startswith("__") and inspect.isfunction(value) + ] + assert own == [] + + def test_the_firewall_role_does_not_carry_config_sync(self): + """A driver mixes the capability in; the role does not promise it.""" + assert not issubclass(FirewallDriver, ConfigSyncMixin) + + def test_a_driver_that_implements_them_is_found_by_hasattr(self): + class Pair(ConfigSyncMixin, DhcpServerMixin): + def get_dhcp_ha(self): + return {"enabled": False, "peers": []} + + def get_config_sync(self): + return {"peer": None, "services": [], "unmapped": []} + + assert callable(getattr(Pair(), "get_dhcp_ha", None)) + assert callable(getattr(Pair(), "get_config_sync", None)) + + +class TestTypes: + def test_are_exported(self): + for name in ("ConfigSyncMixin", "ConfigSyncDict", "DhcpHaDict", "DhcpHaPeerDict"): + assert name in package.__all__ and hasattr(package, name), name + + def test_a_peer_needs_only_its_name_and_role(self): + """A future server may not have a URL to report; Kea does.""" + from napalm_device_types import DhcpHaPeerDict + + assert DhcpHaPeerDict.__required_keys__ == frozenset({"name", "role"}) + + def test_ha_needs_only_whether_it_is_on_and_the_peers(self): + from napalm_device_types import DhcpHaDict + + assert DhcpHaDict.__required_keys__ == frozenset({"enabled", "peers"}) + + def test_config_sync_reports_what_it_could_not_name(self): + """A sync item the driver has no service name for is still a fact.""" + from napalm_device_types import ConfigSyncDict + + assert ConfigSyncDict.__required_keys__ == frozenset({"peer", "services", "unmapped"}) From b4c4ec0b0bc6d5c111a575a05d7061f40cc3105c Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 16 Oct 2026 12:00:06 +0200 Subject: [PATCH 04/50] feat: SnmpHealthProfile, the OIDs a driver declares; deprecate HealthMetricsMixin (4.12.0) Drivers no longer collect over SNMP (docs/STRUCTURE.md). A driver whose device publishes its health values under OIDs of its own declares them as SNMP_HEALTH_PROFILE: per value either scalars asked in order or a table folded by first, average or max, optionally with a pattern for text answers; memory as total with used or free, or a percentage, in KiB or bytes; and how to read the interfaces. The application polls and computes. HealthMetricsMixin.get_health_metrics now warns that it is deprecated; it goes in 5.0. Closes #32 --- README.md | 17 +++++++++ napalm_device_types/__init__.py | 8 ++++ napalm_device_types/health_metrics.py | 18 ++++++++- napalm_device_types/models.py | 47 ++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_snmp_health_profile.py | 53 +++++++++++++++++++++++++++ 6 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 tests/test_snmp_health_profile.py diff --git a/README.md b/README.md index 817a8fa..d784b09 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,23 @@ is a thin bundle over them — `PackageManagementMixin`, `HealthMetricsMixin`, `HostRebootMixin` (`reboot_host`) is mixed into `DeviceTypeDriver` itself, since any device may be restartable; like the others it only declares. +**SNMP is the application's** (`docs/STRUCTURE.md`). `HealthMetricsMixin` is deprecated +since 4.12 and goes in 5.0. A driver does not collect over SNMP; a driver whose device +publishes its health values under OIDs of its own declares them as data: + +```python +class MySwitchDriver(SwitchDriver): + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = { + "uptime": {"oids": ["1.3.6.1.2.1.1.3.0"]}, + "cpu": {"oids": ["", ""]}, + "memory": {"total": {"oids": [""]}, "free": {"oids": [""]}, "unit": "KiB"}, + } +``` + +A value is `oids` (the first that answers wins) or a `table` folded by `first`, `average` +or `max`, with an optional `pattern` for an agent that answers with text. What the profile +does not name, the application reads from the standard UCD-MIB and IF-MIB. + `KernelFactsMixin` (`get_kernel_facts`) is the exception that is mixed in by a driver rather than by a role base: what a Linux kernel has built and loaded is read the same way everywhere, so the command and its parse are concrete here and a driver supplies only diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index bad9703..60fcabc 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -108,6 +108,10 @@ from napalm_device_types.models import ( ManagementProtocolsDict, PowerAction, PowerStateDict, + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, ) from napalm_device_types.nat_vpn import NatVpnMixin from napalm_device_types.packages import PackageManagementMixin @@ -153,6 +157,10 @@ from napalm_device_types.storage import StorageDriver from napalm_device_types.switch import SwitchDriver __all__ = [ + "SnmpHealthProfile", + "SnmpInterfacesSpec", + "SnmpMemorySpec", + "SnmpValueSpec", "AccessPointDriver", "ByteStream", "CommandChannelMixin", diff --git a/napalm_device_types/health_metrics.py b/napalm_device_types/health_metrics.py index 631251d..50a54a5 100644 --- a/napalm_device_types/health_metrics.py +++ b/napalm_device_types/health_metrics.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- """SNMP health collection, shared by every device type that speaks UCD-MIB. +**Deprecated** (docs/STRUCTURE.md): drivers do not collect over SNMP. The +application polls; a driver whose device publishes values under its own OIDs +declares them as ``SNMP_HEALTH_PROFILE`` (:class:`~napalm_device_types.models.SnmpHealthProfile`). +This mixin goes in 5.0. + The collection itself is generic: CPU, memory, uptime, load and per-interface counters come from the same standard OIDs on any UCD-MIB/IF-MIB capable device. Only two details vary by device type, and both are class attributes rather than @@ -15,6 +20,7 @@ implement ``get_health_metrics`` themselves. from __future__ import annotations import re +import warnings from typing import Any, Awaitable, Callable, ClassVar, Dict, cast from napalm_device_types._ucd_metrics import IF_SKIP_DEFAULT, collect_ucd_metrics @@ -39,7 +45,17 @@ class HealthMetricsMixin: snmp_get: Callable[..., Awaitable[Any]], snmp_walk: Callable[..., Awaitable[Dict[str, Any]]], ) -> HealthMetricsDict: - """Collect SNMP health metrics (CPU, memory, uptime, load, interfaces).""" + """Collect SNMP health metrics (CPU, memory, uptime, load, interfaces). + + Deprecated: the application polls; see the module docstring. + """ + warnings.warn( + "HealthMetricsMixin.get_health_metrics is deprecated and goes in 5.0: " + "the application polls over SNMP; a driver declares only its OIDs, " + "as SNMP_HEALTH_PROFILE", + DeprecationWarning, + stacklevel=2, + ) # collect_ucd_metrics is untyped shared plumbing; the shape it returns is # the contract HealthMetricsDict describes. return cast( diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 5dff889..6be7d50 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -993,6 +993,53 @@ class HealthMetricsDict(TypedDict): interfaces: NotRequired[Dict[str, HealthMetricsIfaceDict]] +class SnmpValueSpec(TypedDict, total=False): + """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. + + Either ``oids``, scalars asked in order (the first non-empty answer wins), or + ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a + regular expression whose first group is the number, for an agent that answers + with text. + """ + + oids: List[str] + table: str + fold: Literal["first", "average", "max"] + pattern: str + + +class SnmpMemorySpec(TypedDict, total=False): + """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" + + total: SnmpValueSpec + used: SnmpValueSpec + free: SnmpValueSpec + percent: SnmpValueSpec + unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default + + +class SnmpInterfacesSpec(TypedDict, total=False): + """How the IF-MIB counters of a device are to be read.""" + + skip: str # a pattern of interface names whose counters are noise, not signal + tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) + + +class SnmpHealthProfile(TypedDict, total=False): + """Where a device's SNMP agent publishes its health values. + + A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer + the standard UCD-MIB and IF-MIB; the application polls and computes. What a + profile does not name is read the standard way. Uptime is in TimeTicks, CPU in + percent busy. + """ + + uptime: SnmpValueSpec + cpu: SnmpValueSpec + memory: SnmpMemorySpec + interfaces: SnmpInterfacesSpec + + class NICConfigDict(TypedDict): """Network interface configuration for VM provisioning.""" diff --git a/pyproject.toml b/pyproject.toml index 4cc9830..2923ea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.11.0" +version = "4.12.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_snmp_health_profile.py b/tests/test_snmp_health_profile.py new file mode 100644 index 0000000..c5b306b --- /dev/null +++ b/tests/test_snmp_health_profile.py @@ -0,0 +1,53 @@ +"""SNMP health: drivers declare data, the application polls (docs/STRUCTURE.md). + +``SnmpHealthProfile`` is the shape a driver states its device's OIDs in. +``HealthMetricsMixin``, which collected in the base package, is deprecated. +""" + +from __future__ import annotations + +import asyncio +from typing import ClassVar, get_type_hints + +import pytest + +from napalm_device_types import ( + HealthMetricsMixin, + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, +) + + +def test_a_profile_says_where_each_value_lives(): + assert set(get_type_hints(SnmpHealthProfile)) == {"uptime", "cpu", "memory", "interfaces"} + assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern"} + assert set(get_type_hints(SnmpMemorySpec)) == {"total", "used", "free", "percent", "unit"} + assert set(get_type_hints(SnmpInterfacesSpec)) == {"skip", "tx_errors_are_drops"} + + +def test_every_key_is_optional(): + for spec in (SnmpHealthProfile, SnmpValueSpec, SnmpMemorySpec, SnmpInterfacesSpec): + assert spec.__required_keys__ == frozenset(), spec.__name__ + + +def test_a_driver_declares_its_profile_as_a_class_attribute(): + class Driver: + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = { + "cpu": {"oids": ["1.3.6.1.4.1.9.1", "1.3.6.1.4.1.9.2"]}, + "memory": {"percent": {"table": "1.3.6.1.4.1.9.3", "fold": "first"}}, + } + + assert Driver.SNMP_HEALTH_PROFILE["cpu"]["oids"][0] == "1.3.6.1.4.1.9.1" + + +def test_the_collector_in_the_base_package_warns_that_it_is_deprecated(): + async def get(oid): + return None + + async def walk(oid): + return {} + + with pytest.warns(DeprecationWarning, match="SNMP_HEALTH_PROFILE"): + asyncio.run(HealthMetricsMixin.get_health_metrics(get, walk)) From 9324a5b8f101b932bbb37356b3e494ebdc1f7323 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 16 Oct 2026 15:29:42 +0200 Subject: [PATCH 05/50] feat: fix routines as templates with the caller's values; deprecate run_device_action (4.13.0) run_device_action was a string dispatch drivers filled with an application's remediations, values hard-coded inside (community "public", the caller's address, a /24 firewall rule). napalm_device_types.fixes declares them as templates instead. A template runs declared steps in order and stops at the first failure (run_fix_steps); the driver implements the steps, or overrides the whole method where its device takes the change in one call; every value comes from the caller. - SnmpAgentMixin.configure_snmp_agent(settings): install, configure, firewall, start. The caller verifies with its own SNMP client. get_management_peer() (optional) says the address the device sees the management connection come from, for choosing allowed_sources. - EnsurePackageMixin.ensure_package(name): install only what is missing. - PackageProxyMixin.configure_package_proxy(url): declared. - ContainerEngineMixin.grant_container_engine_access(user): declared. None is mixed into a role base, so a capability check stays truthful. Closes #33 --- README.md | 9 ++ napalm_device_types/__init__.py | 18 +++ napalm_device_types/container_engine.py | 8 +- napalm_device_types/fixes.py | 151 ++++++++++++++++++++ napalm_device_types/models.py | 33 +++++ napalm_device_types/os.py | 5 + pyproject.toml | 2 +- tests/test_fix_templates.py | 174 ++++++++++++++++++++++++ 8 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 napalm_device_types/fixes.py create mode 100644 tests/test_fix_templates.py diff --git a/README.md b/README.md index d784b09..e6fb8e3 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,15 @@ is a thin bundle over them — `PackageManagementMixin`, `HealthMetricsMixin`, `HostRebootMixin` (`reboot_host`) is mixed into `DeviceTypeDriver` itself, since any device may be restartable; like the others it only declares. +**Fixes are templates** (`napalm_device_types.fixes`, since 4.13). A change the +application wants made on a device runs as fixed steps the driver implements, with values +the caller supplies: `SnmpAgentMixin.configure_snmp_agent(settings)` (install, configure, +firewall, start), `EnsurePackageMixin.ensure_package(name)`, +`PackageProxyMixin.configure_package_proxy(url)` and +`ContainerEngineMixin.grant_container_engine_access(user)`. A driver whose device takes the +change in one call overrides the template method. A driver mixes in the fixes its device +can take; none is on a role base. They replace `run_device_action`, which is deprecated. + **SNMP is the application's** (`docs/STRUCTURE.md`). `HealthMetricsMixin` is deprecated since 4.12 and goes in 5.0. A driver does not collect over SNMP; a driver whose device publishes its health values under OIDs of its own declares them as data: diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 60fcabc..7f942a6 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -83,6 +83,12 @@ from napalm_device_types.firewall import FirewallDriver from napalm_device_types.hypervisor import HypervisorDriver from napalm_device_types.os import OSDriver from napalm_device_types.firewall_rules import FirewallRuleMixin +from napalm_device_types.fixes import ( + EnsurePackageMixin, + PackageProxyMixin, + SnmpAgentMixin, + run_fix_steps, +) from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.host_reboot import HostRebootMixin from napalm_device_types.interface_filter import InterfaceFilterMixin @@ -108,6 +114,10 @@ from napalm_device_types.models import ( ManagementProtocolsDict, PowerAction, PowerStateDict, + FixResultDict, + FixStepDict, + FixStepOutcomeDict, + SnmpAgentSettingsDict, SnmpHealthProfile, SnmpInterfacesSpec, SnmpMemorySpec, @@ -157,6 +167,14 @@ from napalm_device_types.storage import StorageDriver from napalm_device_types.switch import SwitchDriver __all__ = [ + "EnsurePackageMixin", + "FixResultDict", + "FixStepDict", + "FixStepOutcomeDict", + "PackageProxyMixin", + "SnmpAgentMixin", + "SnmpAgentSettingsDict", + "run_fix_steps", "SnmpHealthProfile", "SnmpInterfacesSpec", "SnmpMemorySpec", diff --git a/napalm_device_types/container_engine.py b/napalm_device_types/container_engine.py index 265940a..743ab1d 100644 --- a/napalm_device_types/container_engine.py +++ b/napalm_device_types/container_engine.py @@ -32,7 +32,7 @@ import shlex from typing import Any, Dict, List, Optional, Sequence, TYPE_CHECKING, cast from napalm_device_types.channel import ByteStream, CommandResult -from napalm_device_types.models import ContainerEngineDict +from napalm_device_types.models import ContainerEngineDict, FixResultDict #: Engines this package knows how to reach, and the API each speaks. ENGINES: Dict[str, str] = {"docker": "docker-engine"} @@ -111,6 +111,12 @@ class ContainerEngineMixin: def open_stream(self, command: str, *, privileged: bool = False) -> ByteStream: ... + def grant_container_engine_access(self, user: str) -> FixResultDict: + """Let *user* reach the container engine without privilege (the + ``docker`` group on most hosts). A fix template: see + :mod:`napalm_device_types.fixes`. Declared only; a driver implements it.""" + ... + def _container_engine_binary(self, engine: str) -> str: """The host command that is *engine*'s CLI. Override where it is not on PATH.""" return engine diff --git a/napalm_device_types/fixes.py b/napalm_device_types/fixes.py new file mode 100644 index 0000000..33c2ea0 --- /dev/null +++ b/napalm_device_types/fixes.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +"""Fix routines: changes an application wants made on a device, as templates. + +A template fixes the order of the steps; the driver knows how each step is done on +its device; the caller supplies every value -- the community, the allowed sources, +the proxy URL, the user. A driver whose device takes the whole change in one call +(a REST API) overrides the template method instead of its steps. + +None of these is mixed into a role base. A driver mixes in the fixes its device can +take, so ``callable(getattr(driver, "configure_snmp_agent", None))`` stays a truthful +answer; a driver that inherits one it cannot run sets it to ``None``. + +They replace ``run_device_action`` (docs/STRUCTURE.md): + +========================== =================================================== +``fix_snmp`` :meth:`SnmpAgentMixin.configure_snmp_agent` +``fix_apt_proxy`` :meth:`PackageProxyMixin.configure_package_proxy` +``fix_docker_permissions`` ``ContainerEngineMixin.grant_container_engine_access`` +``install_lldpd`` and co. :meth:`EnsurePackageMixin.ensure_package` +========================== =================================================== +""" + +from __future__ import annotations + +from typing import Any, Callable, List, Optional, Sequence, Tuple, TYPE_CHECKING + +from napalm_device_types.models import ( + FixResultDict, + FixStepDict, + FixStepOutcomeDict, + PackageDict, + SnmpAgentSettingsDict, +) + +#: One step of a fix: its name and the call that does it. +FixStep = Tuple[str, Callable[[], FixStepOutcomeDict]] + + +def run_fix_steps(steps: Sequence[FixStep]) -> FixResultDict: + """Run *steps* in order and stop at the first that fails. + + A step that raises has failed, with the exception's message as its output. + """ + done: List[FixStepDict] = [] + for name, step in steps: + try: + outcome = step() + ok, output = bool(outcome["ok"]), str(outcome.get("output", "")) + except Exception as exc: # noqa: BLE001 - any failure of a step ends the fix + ok, output = False, str(exc) + done.append({"step": name, "ok": ok, "output": output}) + if not ok: + break + return { + "success": bool(done) and all(s["ok"] for s in done) and len(done) == len(steps), + "steps": done, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in done), + } + + +class SnmpAgentMixin: + """Run an SNMP agent on the device that answers the caller. + + The steps: install the agent, write its configuration, let the allowed sources + through the host's firewall, start it. Whether it answers is the caller's to ask, + with its own SNMP client. + """ + + def configure_snmp_agent(self, settings: SnmpAgentSettingsDict) -> FixResultDict: + """Install, configure, open and start the agent with *settings*.""" + return run_fix_steps( + [ + ("install", self._install_snmp_agent), + ("configure", lambda: self._write_snmp_agent_config(settings)), + ("firewall", lambda: self._allow_snmp_sources(settings)), + ("start", self._start_snmp_agent), + ] + ) + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_management_peer(self) -> Optional[str]: + """The address the device sees this management connection come from + (for SSH, the client side of ``SSH_CONNECTION``), or None when it cannot + tell. A fact for the caller choosing ``allowed_sources``; optional -- + a driver that cannot tell does not implement it.""" + ... + + # The steps a driver implements: + + def _install_snmp_agent(self) -> FixStepOutcomeDict: ... + + def _write_snmp_agent_config(self, settings: SnmpAgentSettingsDict) -> FixStepOutcomeDict: ... + + def _allow_snmp_sources(self, settings: SnmpAgentSettingsDict) -> FixStepOutcomeDict: + """Let ``settings["allowed_sources"]`` reach the agent; ``ok`` with nothing + to do where the host has no firewall of its own.""" + ... + + def _start_snmp_agent(self) -> FixStepOutcomeDict: + """(Re)start the agent so it reads the new configuration, and enable it at boot.""" + ... + + +class EnsurePackageMixin: + """Make sure a package is installed, installing it only where it is missing. + + Works over :class:`~napalm_device_types.packages.PackageManagementMixin`'s + ``get_packages`` and ``install_package``, which the driver implements. + """ + + def ensure_package(self, name: str) -> FixResultDict: + """Install *name* unless it is installed already.""" + + def install() -> FixStepOutcomeDict: + if any(p.get("name") == name and p.get("installed") for p in self.get_packages()): + return {"ok": True, "output": f"{name} is already installed"} + result: Any = self.install_package(name) # type: ignore[func-returns-value] + # The contract says None (and raise on failure); some drivers still + # return {"success", "output"}. + if isinstance(result, dict): + return {"ok": bool(result.get("success")), "output": str(result.get("output", ""))} + return {"ok": True, "output": f"installed {name}"} + + return run_fix_steps([("install", install)]) + + if TYPE_CHECKING: # pragma: no cover + + def get_packages(self) -> List[PackageDict]: ... + + def install_package(self, name: str, version: str = "") -> None: ... + + +class PackageProxyMixin: + """Point the device's package manager at a caching proxy.""" + + if TYPE_CHECKING: # pragma: no cover + + def configure_package_proxy(self, url: str) -> FixResultDict: + """Have the package manager fetch through *url*; an empty *url* removes it.""" + ... + + +__all__ = [ + "EnsurePackageMixin", + "FixStep", + "PackageProxyMixin", + "SnmpAgentMixin", + "run_fix_steps", +] + diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 6be7d50..756d806 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -993,6 +993,39 @@ class HealthMetricsDict(TypedDict): interfaces: NotRequired[Dict[str, HealthMetricsIfaceDict]] +class FixStepOutcomeDict(TypedDict): + """What one step of a fix routine reports (see ``napalm_device_types.fixes``).""" + + ok: bool + output: str + + +class FixStepDict(TypedDict): + """One step of a fix routine, as ``FixResultDict`` lists it.""" + + step: str + ok: bool + output: str + + +class FixResultDict(TypedDict): + """The result of a fix routine: every step that ran, up to the first failure.""" + + success: bool + steps: List[FixStepDict] + output: str # the steps' output, one "[step] output" line each + + +class SnmpAgentSettingsDict(TypedDict): + """What the caller wants an SNMP agent to be (``SnmpAgentMixin``).""" + + community: str # the read-only v2c community + allowed_sources: List[str] # addresses or CIDRs let through to the agent; [] changes no firewall + contact: NotRequired[str] # sysContact + location: NotRequired[str] # sysLocation + listen: NotRequired[str] # the address the agent binds to; absent: every address + + class SnmpValueSpec(TypedDict, total=False): """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. diff --git a/napalm_device_types/os.py b/napalm_device_types/os.py index 41785d0..27f7287 100644 --- a/napalm_device_types/os.py +++ b/napalm_device_types/os.py @@ -201,6 +201,11 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM """ Executes a named administrative action on the device. + **Deprecated** (docs/STRUCTURE.md): the actions become fix templates + in :mod:`napalm_device_types.fixes`, whose values the caller supplies -- + ``configure_snmp_agent``, ``configure_package_proxy``, + ``grant_container_engine_access``, ``ensure_package``. Goes in 5.0. + This method is an extensibility point for driver-specific one-off operations that do not fit any other NAPALM API method. Each driver documents the action names it supports. diff --git a/pyproject.toml b/pyproject.toml index 2923ea5..ada551f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.12.0" +version = "4.13.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_fix_templates.py b/tests/test_fix_templates.py new file mode 100644 index 0000000..886a975 --- /dev/null +++ b/tests/test_fix_templates.py @@ -0,0 +1,174 @@ +"""Fix routines are templates (docs/STRUCTURE.md, NetOrk/netork#1105). + +The template fixes the order of the steps, the driver knows how each step is done +on its device, the caller supplies every value. None of them is mixed into a role +base, so ``callable(getattr(driver, name, None))`` stays a truthful answer. +""" + +from __future__ import annotations + +import pytest + +from napalm_device_types import ( + AccessPointDriver, + EnsurePackageMixin, + FirewallDriver, + HypervisorDriver, + ManagementControllerDriver, + MediaDriver, + OSDriver, + PhoneDriver, + ResidentialGatewayDriver, + StorageDriver, + SwitchDriver, + PackageProxyMixin, + SnmpAgentMixin, + run_fix_steps, +) + +ROLE_BASES = [ + AccessPointDriver, FirewallDriver, HypervisorDriver, ManagementControllerDriver, MediaDriver, + OSDriver, PhoneDriver, ResidentialGatewayDriver, StorageDriver, SwitchDriver, +] # fmt: skip + + +def _ok(output: str = ""): + return lambda: {"ok": True, "output": output} + + +def _fail(output: str): + return lambda: {"ok": False, "output": output} + + +class TestRunFixSteps: + def test_every_step_runs_in_order_and_is_reported(self): + result = run_fix_steps([("install", _ok("installed")), ("start", _ok("started"))]) + + assert result["success"] is True + assert result["steps"] == [ + {"step": "install", "ok": True, "output": "installed"}, + {"step": "start", "ok": True, "output": "started"}, + ] + assert result["output"] == "[install] installed\n[start] started" + + def test_the_first_failure_stops_the_rest(self): + ran = [] + + def later(): + ran.append("later") + return {"ok": True, "output": ""} + + result = run_fix_steps([("install", _fail("no package")), ("start", later)]) + + assert result["success"] is False + assert [s["step"] for s in result["steps"]] == ["install"] + assert ran == [] + + def test_a_step_that_raises_is_a_failed_step(self): + def broken(): + raise RuntimeError("connection lost") + + result = run_fix_steps([("install", broken)]) + + assert result["success"] is False + assert result["steps"] == [{"step": "install", "ok": False, "output": "connection lost"}] + + +class _Agent(SnmpAgentMixin): + def __init__(self, fail_at: str = ""): + self.calls: list = [] + self.fail_at = fail_at + + def _step(self, name, *args): + self.calls.append((name, *args)) + return {"ok": name != self.fail_at, "output": name} + + def _install_snmp_agent(self): + return self._step("install") + + def _write_snmp_agent_config(self, settings): + return self._step("configure", settings) + + def _allow_snmp_sources(self, settings): + return self._step("firewall", settings) + + def _start_snmp_agent(self): + return self._step("start") + + +class TestSnmpAgent: + SETTINGS = {"community": "c0mm", "allowed_sources": ["10.0.0.5"], "contact": "ops", "location": "rack 1"} + + def test_install_configure_open_and_start_with_the_caller_s_values(self): + agent = _Agent() + + result = agent.configure_snmp_agent(self.SETTINGS) + + assert result["success"] is True + assert agent.calls == [ + ("install",), + ("configure", self.SETTINGS), + ("firewall", self.SETTINGS), + ("start",), + ] + + def test_a_failed_install_configures_nothing(self): + agent = _Agent(fail_at="install") + + assert agent.configure_snmp_agent(self.SETTINGS)["success"] is False + assert agent.calls == [("install",)] + + +class _Packages(EnsurePackageMixin): + def __init__(self, installed): + self.installed = installed + self.asked: list = [] + + def get_packages(self): + return [{"name": n, "installed": True} for n in self.installed] + [ + {"name": "lldpd", "installed": False} + ] + + def install_package(self, name, version=""): + self.asked.append(name) + + +class TestEnsurePackage: + def test_installs_what_is_missing(self): + driver = _Packages(installed=["curl"]) + + result = driver.ensure_package("lldpd") + + assert result["success"] is True + assert driver.asked == ["lldpd"] + + def test_leaves_alone_what_is_there(self): + driver = _Packages(installed=["lldpd"]) + + result = driver.ensure_package("lldpd") + + assert result["success"] is True + assert driver.asked == [] + assert "already installed" in result["output"] + + def test_an_install_that_reports_failure_fails(self): + driver = _Packages(installed=[]) + driver.install_package = lambda name, version="": {"success": False, "output": "E: no candidate"} + + result = driver.ensure_package("lldpd") + + assert result["success"] is False + assert "no candidate" in result["output"] + + +@pytest.mark.parametrize("base", ROLE_BASES, ids=lambda b: b.__name__) +@pytest.mark.parametrize( + "method", + ["configure_snmp_agent", "ensure_package", "configure_package_proxy", "grant_container_engine_access"], +) +def test_no_role_base_offers_a_fix(base, method): + assert getattr(base, method, None) is None + + +def test_the_package_proxy_is_declared_only(): + assert not hasattr(PackageProxyMixin, "configure_package_proxy") From 2805f911c4fadbaee93d29cec766cbd400cdda20 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 17 Oct 2026 09:12:20 +0200 Subject: [PATCH 06/50] feat: licenses, a hypervisor's hosts and the package proxy as facts (4.14.0) Drivers report facts and the application decides what is a warning (docs/STRUCTURE.md, NetOrk/netork#1111). Three facts drivers so far only reported as warning codes get contracts: - LicenseMixin.get_licenses(): licenses and subscriptions with their status (a Proxmox subscription, a VMware edition, a BMC license a feature needs). - HypervisorDriver.get_hypervisor_hosts(): each host connected, in maintenance, with configuration issues, under a central manager. - PackageProxyMixin.get_package_proxy(): the proxy apt fetches through. All declared only. --- napalm_device_types/__init__.py | 6 +++++ napalm_device_types/fixes.py | 6 +++++ napalm_device_types/hypervisor.py | 7 ++++++ napalm_device_types/licenses.py | 27 +++++++++++++++++++++ napalm_device_types/models.py | 27 +++++++++++++++++++++ pyproject.toml | 2 +- tests/test_fact_getters.py | 40 +++++++++++++++++++++++++++++++ 7 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/licenses.py create mode 100644 tests/test_fact_getters.py diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 7f942a6..8a9a5d6 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -90,6 +90,7 @@ from napalm_device_types.fixes import ( run_fix_steps, ) from napalm_device_types.health_metrics import HealthMetricsMixin +from napalm_device_types.licenses import LicenseMixin from napalm_device_types.host_reboot import HostRebootMixin from napalm_device_types.interface_filter import InterfaceFilterMixin from napalm_device_types.kernel import KERNEL_FACTS_COMMAND, KernelFactsMixin, parse_kernel_facts @@ -114,6 +115,8 @@ from napalm_device_types.models import ( ManagementProtocolsDict, PowerAction, PowerStateDict, + HypervisorHostDict, + LicenseDict, FixResultDict, FixStepDict, FixStepOutcomeDict, @@ -167,6 +170,9 @@ from napalm_device_types.storage import StorageDriver from napalm_device_types.switch import SwitchDriver __all__ = [ + "HypervisorHostDict", + "LicenseDict", + "LicenseMixin", "EnsurePackageMixin", "FixResultDict", "FixStepDict", diff --git a/napalm_device_types/fixes.py b/napalm_device_types/fixes.py index 33c2ea0..9656960 100644 --- a/napalm_device_types/fixes.py +++ b/napalm_device_types/fixes.py @@ -140,6 +140,12 @@ class PackageProxyMixin: """Have the package manager fetch through *url*; an empty *url* removes it.""" ... + def get_package_proxy(self) -> Optional[str]: + """The proxy the package manager fetches through now; None for none. + + A fact: whether it is the right one is the caller's to judge.""" + ... + __all__ = [ "EnsurePackageMixin", diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 32173a4..eb42575 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -16,6 +16,7 @@ from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents from napalm_device_types.models import ( + HypervisorHostDict, NICConfigDict, NetworkTargetDict, SnapshotDict, @@ -637,6 +638,12 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... + def get_hypervisor_hosts(self) -> List[HypervisorHostDict]: + """The hosts this hypervisor or its manager runs, and how each stands: + connected, in maintenance, with configuration issues, under a central + manager. Facts; what is a problem is the caller's to decide.""" + ... + def get_vm_cpu_types(self) -> List[VMCpuTypeDict]: """ List the virtual CPU models a new VM may be given on the node it will diff --git a/napalm_device_types/licenses.py b/napalm_device_types/licenses.py new file mode 100644 index 0000000..df0c672 --- /dev/null +++ b/napalm_device_types/licenses.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +"""Licenses and subscriptions a device reports. + +A fact for the caller (docs/STRUCTURE.md): whether a free edition, a missing +subscription or a license a feature needs is a problem is decided above the +driver. Declared only; a driver that can read licenses implements it. +""" + +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from napalm_device_types.models import LicenseDict + + +class LicenseMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_licenses(self) -> List[LicenseDict]: + """Every license or subscription the device knows of, with its status. + + Example:: + + [{"name": "pve-subscription", "status": "none"}, + {"name": "vSphere", "status": "active", "edition": "Free"}] + """ + ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 756d806..91cbc63 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -993,6 +993,33 @@ class HealthMetricsDict(TypedDict): interfaces: NotRequired[Dict[str, HealthMetricsIfaceDict]] +LicenseStatus = Literal["active", "missing", "expired", "invalid", "none", "unknown"] + + +class LicenseDict(TypedDict): + """A license or subscription the device reports (``LicenseMixin.get_licenses``). + + A fact, not a judgement: whether a free edition or a missing subscription is + a problem is the caller's to decide. + """ + + name: str # what it licenses, as the device names it + status: LicenseStatus # "none": the device runs without one, as installed + edition: NotRequired[str] # the edition or level, e.g. "Free", "Enterprise Plus" + expires: NotRequired[Optional[str]] # ISO date, None for no expiry + detail: NotRequired[str] # the device's own words, e.g. why a feature is refused + + +class HypervisorHostDict(TypedDict): + """One host of a hypervisor or cluster (``HypervisorDriver.get_hypervisor_hosts``).""" + + name: str + connected: Optional[bool] # reachable by its manager; None where it cannot tell + maintenance: Optional[bool] # in maintenance mode + config_issues: List[str] # the manager's configuration warnings, verbatim + managed_by: NotRequired[Optional[str]] # the address of a central manager, if any + + class FixStepOutcomeDict(TypedDict): """What one step of a fix routine reports (see ``napalm_device_types.fixes``).""" diff --git a/pyproject.toml b/pyproject.toml index ada551f..2415910 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.13.0" +version = "4.14.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py new file mode 100644 index 0000000..11ddcca --- /dev/null +++ b/tests/test_fact_getters.py @@ -0,0 +1,40 @@ +"""Facts instead of warnings (docs/STRUCTURE.md, NetOrk/netork#1111). + +A driver reports what it reads; the application decides what is a warning. Three +facts that drivers so far only reported as warning codes get contracts of their +own: the package manager's proxy, licenses and subscriptions, and the state of a +hypervisor's hosts. All are declared only, so a capability check stays truthful. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types import ( + HypervisorDriver, + HypervisorHostDict, + LicenseDict, + LicenseMixin, + PackageProxyMixin, +) + + +def test_a_license_says_what_it_is_and_whether_it_holds(): + hints = get_type_hints(LicenseDict) + assert {"name", "status"} <= set(hints) + assert LicenseDict.__required_keys__ == frozenset({"name", "status"}) + assert set(hints) == {"name", "status", "edition", "expires", "detail"} + + +def test_a_hypervisor_host_says_how_it_stands(): + hints = get_type_hints(HypervisorHostDict) + assert set(hints) == {"name", "connected", "maintenance", "config_issues", "managed_by"} + assert HypervisorHostDict.__required_keys__ == frozenset( + {"name", "connected", "maintenance", "config_issues"} + ) + + +def test_the_getters_are_declared_only(): + assert not hasattr(PackageProxyMixin, "get_package_proxy") + assert not hasattr(LicenseMixin, "get_licenses") + assert not hasattr(HypervisorDriver, "get_hypervisor_hosts") From 2d1c40c87d2dc80f6aa704747aadd18b6851d4ab Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 17 Oct 2026 11:32:10 +0200 Subject: [PATCH 07/50] feat: a missing license can say what the device withholds without it (4.15.0) LicenseDict gets withholds: NotRequired[List[str]]: what a device keeps back for want of a license while the rest of it answers. A Supermicro X10 with only SFT-OOB-LIC answers Redfish but withholds its drives and firmware inventory until DCMS is activated. That makes it partly licensed, not unlicensed, and the caller can tell the two apart (NetOrk/netork#1111). --- napalm_device_types/licenses.py | 7 ++++++- napalm_device_types/models.py | 5 +++++ pyproject.toml | 2 +- tests/test_fact_getters.py | 10 +++++++++- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/napalm_device_types/licenses.py b/napalm_device_types/licenses.py index df0c672..c5a6b83 100644 --- a/napalm_device_types/licenses.py +++ b/napalm_device_types/licenses.py @@ -22,6 +22,11 @@ class LicenseMixin: Example:: [{"name": "pve-subscription", "status": "none"}, - {"name": "vSphere", "status": "active", "edition": "Free"}] + {"name": "vSphere", "status": "active", "edition": "Free"}, + {"name": "DCMS", "status": "missing", + "withholds": ["drives", "firmware inventory"]}] + + ``withholds`` names what a device keeps back for want of a license while + the rest of it answers; a missing license without it may keep back all. """ ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 91cbc63..0a4e9d9 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1008,6 +1008,11 @@ class LicenseDict(TypedDict): edition: NotRequired[str] # the edition or level, e.g. "Free", "Enterprise Plus" expires: NotRequired[Optional[str]] # ISO date, None for no expiry detail: NotRequired[str] # the device's own words, e.g. why a feature is refused + # What the device withholds without it while the rest works, e.g. ["drives", + # "firmware inventory"] on a BMC that answers but wants a further license. + # Absent: the device says nothing of the kind, or withholds everything it + # would offer with it. + withholds: NotRequired[List[str]] class HypervisorHostDict(TypedDict): diff --git a/pyproject.toml b/pyproject.toml index 2415910..9eab2a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.14.0" +version = "4.15.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py index 11ddcca..c59e29d 100644 --- a/tests/test_fact_getters.py +++ b/tests/test_fact_getters.py @@ -23,7 +23,15 @@ def test_a_license_says_what_it_is_and_whether_it_holds(): hints = get_type_hints(LicenseDict) assert {"name", "status"} <= set(hints) assert LicenseDict.__required_keys__ == frozenset({"name", "status"}) - assert set(hints) == {"name", "status", "edition", "expires", "detail"} + assert set(hints) == {"name", "status", "edition", "expires", "detail", "withholds"} + + +def test_a_missing_license_may_say_what_the_device_withholds_without_it(): + # A Supermicro X10 with only SFT-OOB-LIC answers Redfish but keeps its drives and + # firmware inventory until DCMS is activated: partly licensed, not unlicensed. + hints = get_type_hints(LicenseDict) + assert hints["withholds"] == list[str] or str(hints["withholds"]).endswith("List[str]") + assert "withholds" not in LicenseDict.__required_keys__ def test_a_hypervisor_host_says_how_it_stands(): From 634c1424773d5f705c283db2c818c894e0fc91eb Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 17 Oct 2026 11:36:32 +0200 Subject: [PATCH 08/50] test: read the withholds hint the way Python 3.10 reports it --- tests/test_fact_getters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py index c59e29d..b6fa4e7 100644 --- a/tests/test_fact_getters.py +++ b/tests/test_fact_getters.py @@ -30,7 +30,8 @@ def test_a_missing_license_may_say_what_the_device_withholds_without_it(): # A Supermicro X10 with only SFT-OOB-LIC answers Redfish but keeps its drives and # firmware inventory until DCMS is activated: partly licensed, not unlicensed. hints = get_type_hints(LicenseDict) - assert hints["withholds"] == list[str] or str(hints["withholds"]).endswith("List[str]") + # Python 3.10 keeps NotRequired[...] around the type; later versions strip it. + assert "List[str]" in str(hints["withholds"]) assert "withholds" not in LicenseDict.__required_keys__ From 59fb1273e8ba01c04c7bc3515a97615fc9991e45 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 17 Oct 2026 13:00:43 +0200 Subject: [PATCH 09/50] feat: the firmware a vendor offers, as a fact (4.16.0) FirmwareOfferMixin.get_firmware_offer() -> FirmwareOfferDict {current_version, latest_version, newer, download_url?}: what runs and the newest firmware the vendor offers, from its download catalogue or the device's own update check. Comparing vendor version strings stays with the driver, so it says whether the offer is newer; whether that is a warning is the application's call (NetOrk/netork#1111). Declared only. --- napalm_device_types/__init__.py | 4 ++++ napalm_device_types/firmware.py | 32 ++++++++++++++++++++++++++++++++ napalm_device_types/models.py | 10 ++++++++++ pyproject.toml | 2 +- tests/test_fact_getters.py | 15 +++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/firmware.py diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 8a9a5d6..596cc8c 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -90,6 +90,7 @@ from napalm_device_types.fixes import ( run_fix_steps, ) from napalm_device_types.health_metrics import HealthMetricsMixin +from napalm_device_types.firmware import FirmwareOfferMixin from napalm_device_types.licenses import LicenseMixin from napalm_device_types.host_reboot import HostRebootMixin from napalm_device_types.interface_filter import InterfaceFilterMixin @@ -116,6 +117,7 @@ from napalm_device_types.models import ( PowerAction, PowerStateDict, HypervisorHostDict, + FirmwareOfferDict, LicenseDict, FixResultDict, FixStepDict, @@ -171,6 +173,8 @@ from napalm_device_types.switch import SwitchDriver __all__ = [ "HypervisorHostDict", + "FirmwareOfferDict", + "FirmwareOfferMixin", "LicenseDict", "LicenseMixin", "EnsurePackageMixin", diff --git a/napalm_device_types/firmware.py b/napalm_device_types/firmware.py new file mode 100644 index 0000000..aed95e9 --- /dev/null +++ b/napalm_device_types/firmware.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +"""The firmware a vendor offers for a device. + +A fact for the caller (docs/STRUCTURE.md): a driver may know the newer firmware +from the vendor's download catalogue or the device's own update check; whether +running an older one is a problem is decided above the driver. Comparing the +vendor's version strings is vendor knowledge, so the driver says itself whether +the offer is newer. Declared only; a driver that can tell implements it. +""" + +from __future__ import annotations + +from typing import Optional, TYPE_CHECKING + +from napalm_device_types.models import FirmwareOfferDict + + +class FirmwareOfferMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_firmware_offer(self) -> Optional[FirmwareOfferDict]: + """The firmware running and the newest the vendor offers. + + None when the driver cannot tell for this device (an unknown model, + a catalogue that did not answer). + + Example:: + + {"current_version": "1.0.2.10", "latest_version": "1.0.2.12", + "newer": True, "download_url": "https://.../GS308E_1.0.2.12.zip"} + """ + ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 0a4e9d9..cb9e8d9 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1015,6 +1015,16 @@ class LicenseDict(TypedDict): withholds: NotRequired[List[str]] +class FirmwareOfferDict(TypedDict): + """The firmware a device runs and the newest its vendor offers + (``FirmwareOfferMixin.get_firmware_offer``). A fact, not a judgement.""" + + current_version: str + latest_version: str + newer: bool # the offer is newer than what runs, by the vendor's versioning + download_url: NotRequired[str] + + class HypervisorHostDict(TypedDict): """One host of a hypervisor or cluster (``HypervisorDriver.get_hypervisor_hosts``).""" diff --git a/pyproject.toml b/pyproject.toml index 9eab2a9..25af5dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.15.0" +version = "4.16.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py index b6fa4e7..f715bfc 100644 --- a/tests/test_fact_getters.py +++ b/tests/test_fact_getters.py @@ -47,3 +47,18 @@ def test_the_getters_are_declared_only(): assert not hasattr(PackageProxyMixin, "get_package_proxy") assert not hasattr(LicenseMixin, "get_licenses") assert not hasattr(HypervisorDriver, "get_hypervisor_hosts") + + +def test_a_firmware_offer_says_what_runs_and_what_is_offered(): + """The vendor's newer firmware is a fact the driver can know (a download + catalogue, the device's own update check); whether it is a warning is the + application's call. Comparing vendor version strings stays with the driver, + so it says itself whether the offer is newer.""" + from napalm_device_types import FirmwareOfferDict, FirmwareOfferMixin + + hints = get_type_hints(FirmwareOfferDict) + assert set(hints) == {"current_version", "latest_version", "newer", "download_url"} + assert FirmwareOfferDict.__required_keys__ == frozenset( + {"current_version", "latest_version", "newer"} + ) + assert not hasattr(FirmwareOfferMixin, "get_firmware_offer") From 1dd448e1d3c13b53bb6f168b923374483bee25b2 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 18 Oct 2026 15:02:15 +0200 Subject: [PATCH 10/50] feat: public contracts for update progress, DNS forwards and OSPF (4.17.0) An application followed a firewall's upgrade, edited its resolver's forwards and read its OSPF state through the driver's private REST helpers (NetOrk/netork#1113). Declared contracts, neutral shapes: - UpdateMixin.get_update_progress() -> UpdateProgressDict {state, log} for an update apply_updates only starts. - DnsForwardMixin: get_dns_forwards(), set_dns_forward(domain, server, port), remove_dns_forward(domain); the driver applies. - OspfMixin.get_ospf() -> OspfDict {router_id, areas, interfaces}; the routing daemon's format is the driver's to translate. All declared only. --- napalm_device_types/__init__.py | 12 ++++++++ napalm_device_types/dns_forward.py | 35 +++++++++++++++++++++ napalm_device_types/models.py | 42 ++++++++++++++++++++++++++ napalm_device_types/ospf.py | 31 +++++++++++++++++++ napalm_device_types/updates.py | 14 ++++++++- pyproject.toml | 2 +- tests/test_public_contracts_1113.py | 47 +++++++++++++++++++++++++++++ 7 files changed, 181 insertions(+), 2 deletions(-) create mode 100644 napalm_device_types/dns_forward.py create mode 100644 napalm_device_types/ospf.py create mode 100644 tests/test_public_contracts_1113.py diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 596cc8c..9393cc4 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -90,7 +90,9 @@ from napalm_device_types.fixes import ( run_fix_steps, ) from napalm_device_types.health_metrics import HealthMetricsMixin +from napalm_device_types.dns_forward import DnsForwardMixin from napalm_device_types.firmware import FirmwareOfferMixin +from napalm_device_types.ospf import OspfMixin from napalm_device_types.licenses import LicenseMixin from napalm_device_types.host_reboot import HostRebootMixin from napalm_device_types.interface_filter import InterfaceFilterMixin @@ -117,7 +119,11 @@ from napalm_device_types.models import ( PowerAction, PowerStateDict, HypervisorHostDict, + DnsForwardDict, FirmwareOfferDict, + OspfDict, + OspfInterfaceDict, + UpdateProgressDict, LicenseDict, FixResultDict, FixStepDict, @@ -173,7 +179,13 @@ from napalm_device_types.switch import SwitchDriver __all__ = [ "HypervisorHostDict", + "DnsForwardDict", + "DnsForwardMixin", "FirmwareOfferDict", + "OspfDict", + "OspfInterfaceDict", + "OspfMixin", + "UpdateProgressDict", "FirmwareOfferMixin", "LicenseDict", "LicenseMixin", diff --git a/napalm_device_types/dns_forward.py b/napalm_device_types/dns_forward.py new file mode 100644 index 0000000..38a277b --- /dev/null +++ b/napalm_device_types/dns_forward.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +"""Domains a device's DNS resolver forwards to another server. + +An application that makes the device's resolver ask another server for a zone +-- the authority for an internal domain -- edits these. The driver writes them +in its device's terms and applies them; the forwards are the caller's. +Declared only; a driver whose device resolves DNS implements it. +""" + +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from napalm_device_types.models import DnsForwardDict + + +class DnsForwardMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_dns_forwards(self) -> List[DnsForwardDict]: + """Every domain the resolver forwards, enabled or not.""" + ... + + def set_dns_forward(self, domain: str, server: str, port: int = 53) -> None: + """Forward *domain* to *server*: added, or the existing forward of + *domain* changed. Applied before it returns. + + :raises RuntimeError: when the device refuses it. + """ + ... + + def remove_dns_forward(self, domain: str) -> None: + """Stop forwarding *domain*; nothing to do when it is not forwarded. + Applied before it returns.""" + ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index cb9e8d9..a2cf8c7 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1015,6 +1015,48 @@ class LicenseDict(TypedDict): withholds: NotRequired[List[str]] +UpdateState = Literal["idle", "running", "done", "failed"] + + +class UpdateProgressDict(TypedDict): + """Where an update the device runs by itself stands + (``UpdateMixin.get_update_progress``).""" + + state: UpdateState # "idle": none has run since the device started, or none is known + log: str # the device's own log of it, whole; a caller follows it by line count + + +class DnsForwardDict(TypedDict): + """A domain the device's resolver forwards to another server + (``DnsForwardMixin.get_dns_forwards``).""" + + domain: str # without the trailing dot + server: str + port: int + enabled: bool + + +class OspfInterfaceDict(TypedDict): + """OSPF on one interface (``OspfMixin.get_ospf``).""" + + address: str + prefix_length: int + area: str + network_type: str # as the routing daemon names it, e.g. "POINTOPOINT" + authentication: str # as the routing daemon names it, e.g. "MessageDigest"; "" for none + neighbors: int + neighbors_adjacent: int # neighbours with a full adjacency + cost: int + + +class OspfDict(TypedDict): + """The device's OSPF state (``OspfMixin.get_ospf``).""" + + router_id: Optional[str] + areas: List[str] + interfaces: Dict[str, OspfInterfaceDict] # by interface name + + class FirmwareOfferDict(TypedDict): """The firmware a device runs and the newest its vendor offers (``FirmwareOfferMixin.get_firmware_offer``). A fact, not a judgement.""" diff --git a/napalm_device_types/ospf.py b/napalm_device_types/ospf.py new file mode 100644 index 0000000..33a96fe --- /dev/null +++ b/napalm_device_types/ospf.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +"""A device's OSPF state, in neutral terms. + +The routing daemon's own output (FRR, Quagga, a vendor's CLI) is the driver's +to translate; whether an interface without neighbours or without +authentication is a problem is the caller's to decide. +Declared only; a driver whose device runs OSPF implements it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import OspfDict + + +class OspfMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_ospf(self) -> OspfDict: + """Router ID, areas and OSPF per interface. + + Example:: + + {"router_id": "10.255.0.1", "areas": ["0.0.0.0"], + "interfaces": {"wg0": {"address": "10.255.0.1", "prefix_length": 24, + "area": "0.0.0.0", "network_type": "POINTOPOINT", + "authentication": "MessageDigest", "neighbors": 1, + "neighbors_adjacent": 1, "cost": 10}}} + """ + ... diff --git a/napalm_device_types/updates.py b/napalm_device_types/updates.py index 56e7934..90059b4 100644 --- a/napalm_device_types/updates.py +++ b/napalm_device_types/updates.py @@ -15,7 +15,7 @@ from __future__ import annotations from typing import Any, Dict, List, TYPE_CHECKING -from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict +from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict, UpdateProgressDict class UpdateMixin: @@ -61,6 +61,18 @@ class UpdateMixin: """ ... + def get_update_progress(self) -> UpdateProgressDict: + """ + Where an update the device runs by itself stands, for a caller that + follows it: a firewall's firmware upgrade, which :meth:`apply_updates` + only starts. ``state`` is ``"running"`` until it ends in ``"done"`` + or ``"failed"``; ``log`` is the device's whole log of it. + + Declared only: a driver whose :meth:`apply_updates` returns when the + update is over has nothing to report here. + """ + ... + def apply_updates(self, packages: List[str]) -> ApplyUpdatesResultDict: """ Upgrades the given packages to the newest available version. diff --git a/pyproject.toml b/pyproject.toml index 25af5dc..84aa952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.16.0" +version = "4.17.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_public_contracts_1113.py b/tests/test_public_contracts_1113.py new file mode 100644 index 0000000..5a861b5 --- /dev/null +++ b/tests/test_public_contracts_1113.py @@ -0,0 +1,47 @@ +"""Public contracts for what an application reached through driver internals. + +netOrk followed a firewall's upgrade, edited its resolver's forwards and read +its OSPF state through the driver's private REST helpers (NetOrk/netork#1113). +Each becomes a declared method with a neutral shape; the vendor's own format +is translated by the driver. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types import ( + DnsForwardDict, + DnsForwardMixin, + OspfDict, + OspfInterfaceDict, + OspfMixin, + UpdateMixin, + UpdateProgressDict, +) + + +def test_update_progress_is_a_state_and_its_log(): + assert set(get_type_hints(UpdateProgressDict)) == {"state", "log"} + assert not hasattr(UpdateMixin, "get_update_progress") + + +def test_a_dns_forward_names_the_domain_and_where_it_goes(): + assert set(get_type_hints(DnsForwardDict)) == {"domain", "server", "port", "enabled"} + for name in ("get_dns_forwards", "set_dns_forward", "remove_dns_forward"): + assert not hasattr(DnsForwardMixin, name), name + + +def test_ospf_in_neutral_terms(): + assert set(get_type_hints(OspfDict)) == {"router_id", "areas", "interfaces"} + assert set(get_type_hints(OspfInterfaceDict)) == { + "address", + "prefix_length", + "area", + "network_type", + "authentication", + "neighbors", + "neighbors_adjacent", + "cost", + } + assert not hasattr(OspfMixin, "get_ospf") From 2bd7bcc77c601808402bca9346a5064269c2e7f3 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 18 Oct 2026 15:28:34 +0200 Subject: [PATCH 11/50] feat: the configuration that says which services run together (4.17.0) ServiceClustersMixin.get_service_clusters() -> ServiceClustersDict {patroni, adguardhome_sync, bunkerweb}: per mechanism the clusters the host's configuration says it belongs to, each with its members (address, port, name, role as the mechanism words it, this_host, config_from) and the unit or container it was read from (source). - Patroni: scope, name and restapi.connect_address from its YAML, then the members and roles /cluster lists from the DCS. - adguardhome-sync: ORIGIN_URL and REPLICA*_URL; the origin pushes to each replica. - BunkerWeb: a worker's API_WHITELIST_IP names the manager it admits, a manager's BUNKERWEB_INSTANCES its workers. Two hosts answering at one virtual IP prove nothing; this is the evidence that they run one service (NetOrk/netork#1125). One generic command for every Linux host, filtered on the host like the virtual IPs' one: allowlisted fields only, URLs without user:password@, DCS endpoints stay there. Root first, then without it; None per mechanism is "did not look". --- README.md | 16 + napalm_device_types/__init__.py | 11 + napalm_device_types/models.py | 62 +++ napalm_device_types/service_clusters.py | 528 ++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_service_clusters.py | 491 ++++++++++++++++++++++ 6 files changed, 1109 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/service_clusters.py create mode 100644 tests/test_service_clusters.py diff --git a/README.md b/README.md index e6fb8e3..c12f40d 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,22 @@ A mechanism it could not read completely is `None` ("did not look"), a host with `[]`. The command is about 4 kB, so a driver sends it on an exec channel (`_run_virtual_ips_command(command, privileged=)`), not typed into a shell. +`ServiceClustersMixin` (`get_service_clusters`) reads the configuration that proves two +hosts run one service together -- not merely that they answer at one address: Patroni +(`scope`, `name` and `restapi.connect_address` from its YAML, then the members and roles +its REST API's `/cluster` lists from the DCS), adguardhome-sync (`ORIGIN_URL` and +`REPLICA*_URL` of its container: the origin pushes to each replica) and BunkerWeb (a +worker's `API_WHITELIST_IP` names the manager it takes its configuration from, a manager's +`BUNKERWEB_INSTANCES` its workers). Each member carries its role as the mechanism words it +and the index of the member it takes its configuration from (`config_from`). The same +configuration holds the superuser password, the sync's logins and BunkerWeb's database URI, +so awk programs filter **on the host**: allowlisted fields only, a URL without its +`user:password@`, and the DCS endpoints stay on the host. Root first (Patroni's YAML is the +postgres user's), then without it. `None` per mechanism is "did not look", `[]` "has none". +The container engine is asked through `_container_engine_binary` where the driver has one. +A driver sends the command on an exec channel (`_run_service_clusters_command(command, +privileged=)`). + `get_dhcp_ha` (declared on `DhcpServerMixin`, optional) is what a DHCP server's own HA configuration says: whether HA is on, which peer this server is, each peer's name, role -- as the server words it, not a shared vocabulary -- and, where the server has one, the URL diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 596cc8c..4b1cfd6 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -51,6 +51,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.nat_vpn.NatVpnMixin` * :class:`~napalm_device_types.packages.PackageManagementMixin` * :class:`~napalm_device_types.ping_sweep.PingSweepMixin` +* :class:`~napalm_device_types.service_clusters.ServiceClustersMixin` * :class:`~napalm_device_types.services.ServiceControlMixin` * :class:`~napalm_device_types.systemd.SystemdServicesMixin` * :class:`~napalm_device_types.updates.UpdateMixin` @@ -155,6 +156,12 @@ from napalm_device_types.system_identity import ( is_placeholder_identity, parse_system_identity, ) +from napalm_device_types.service_clusters import ( + SERVICE_CLUSTERS_COMMAND, + ServiceClustersMixin, + parse_service_clusters, + service_clusters_command, +) from napalm_device_types.virtual_ips import ( VIRTUAL_IPS_COMMAND, VirtualIpsMixin, @@ -256,6 +263,10 @@ __all__ = [ "UpdateMixin", "VIRTUAL_IPS_COMMAND", "VirtualIpsMixin", + "SERVICE_CLUSTERS_COMMAND", + "ServiceClustersMixin", + "parse_service_clusters", + "service_clusters_command", "SystemIdentityMixin", "SYSTEM_IDENTITY_COMMAND", "is_placeholder_identity", diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index cb9e8d9..7cbe4d9 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -425,6 +425,68 @@ class VirtualIpDict(TypedDict): holds: NotRequired[Optional[bool]] +class ServiceClusterMemberDict(TypedDict): + """One member of a service cluster, as the configuration names it. + + A member is named by what the configuration has: an address (a URL's host, a + whitelisted address -- possibly a host name), a port where it gives one, a + name where the mechanism has its own (Patroni's). ``this_host`` marks the + member the reading host is when the configuration does not name it by + address. Only these fields leave the host -- never a password or a DCS + endpoint. + """ + + name: Optional[str] + address: Optional[str] + port: Optional[int] + #: as the mechanism words it: Patroni's "leader"/"sync_standby"/"replica", + #: adguardhome-sync's "origin"/"replica", BunkerWeb's "manager"/"worker" + role: Optional[str] + #: the member's state where the mechanism reports one (Patroni's "running", + #: "streaming"); configuration alone has none + state: Optional[str] + this_host: bool + #: the index in ``members`` of the member whose configuration this one takes: + #: a replica's origin, a worker's manager. None where it takes none. + config_from: Optional[int] + + +class ServiceClusterDict(TypedDict): + """A cluster the host's configuration says it belongs to (netOrk #1125). + + The evidence that two hosts run one service together, not that they merely + answer at one address: Patroni's scope and the members its DCS lists, the + origin and replicas adguardhome-sync keeps in step, a BunkerWeb worker's + manager. + """ + + mechanism: str # "patroni" | "adguardhome-sync" | "bunkerweb" + #: what the members run, as a host reports it: "patroni", "adguardhome", "bunkerweb" + service: str + #: what the evidence was read from on this host: the unit without ".service" + #: ("patroni", "patroni@16-main") or the container's name ("adguardhome-sync") + source: str + #: what the members share where the reading host can name it: Patroni's scope, + #: the sync's origin ("10.9.100.5:3000"), a worker's manager address. None for a + #: manager, which cannot name its own address. + cluster: Optional[str] + running: Optional[bool] # the unit is active; a listed container runs + members: List[ServiceClusterMemberDict] + + +class ServiceClustersDict(TypedDict): + """What a driver's ``get_service_clusters`` read, per mechanism. + + A list is what was looked at -- ``[]`` a host without that mechanism. None is + "did not look": a configuration it could not read, a host without systemd, a + container engine it may not ask, a sync configured by file. + """ + + patroni: Optional[List[ServiceClusterDict]] + adguardhome_sync: Optional[List[ServiceClusterDict]] + bunkerweb: Optional[List[ServiceClusterDict]] + + class VirtualIpsDict(TypedDict): """What a driver's ``get_virtual_ips`` read, per mechanism. diff --git a/napalm_device_types/service_clusters.py b/napalm_device_types/service_clusters.py new file mode 100644 index 0000000..fc22784 --- /dev/null +++ b/napalm_device_types/service_clusters.py @@ -0,0 +1,528 @@ +# -*- coding: utf-8 -*- +"""Service clusters: the configuration that says which services run together. + +Two hosts that answer at one virtual IP may run the same service by accident. +What proves a cluster is the configuration its members share. That is read the +same way on every Linux host, so the command and its parse live here once, and +a driver only carries the command across (netOrk #1125). + +**What is read.** + +- *Patroni*: every ``patroni.service`` and ``patroni@*.service`` unit systemd + knows. Its configuration file -- the unit's first non-flag argument, else + ``/etc/patroni/config.yml`` -- gives ``scope``, ``name`` and + ``restapi.connect_address``. The REST API there (``/cluster``, which Patroni + answers without a login) lists the members its DCS knows, each with its role: + ``leader``, ``sync_standby``, ``replica``. When it does not answer, the host + is a member of its scope on its own. +- *adguardhome-sync*: a running container whose image is adguardhome-sync. + ``ORIGIN_URL`` and ``REPLICA*_URL`` name the AdGuard Home instances it keeps in + step: the origin pushes to every replica. A sync configured by a YAML file + instead of its environment is not read. +- *BunkerWeb*: a running instance container (``bunkerity/bunkerweb``) admits a + manager's configuration pushes from the addresses in ``API_WHITELIST_IP``; a + manager (``bunkerweb-all-in-one``, ``bunkerweb-scheduler``) names its + instances in ``BUNKERWEB_INSTANCES``. BunkerWeb's own default whitelist, + loopback and the docker network, names no one. A manager that keeps its + instances in its database names none here: its workers name it. + +**Filtered on the host.** The same configuration holds secrets -- Patroni's +superuser and REST passwords, the sync's AdGuard logins, BunkerWeb's database +URI. The command never prints a configuration: awk programs print allowlisted +fields only, and a URL loses its ``user:password@``. The DCS endpoints stay on +the host too; the members come from ``/cluster``. + +**None is "did not look".** A Patroni configuration that exists but cannot be +read, a host without systemd, a docker the login may not ask, a sync configured +by file: that mechanism is None, and the caller keeps what it knew. ``[]`` is a +host that has none. + +**Root, and without it.** Patroni's configuration is the postgres user's alone. +The whole script goes to the host as one ``sh -c`` argument; when the privileged +call brings no report back, it runs again without privilege. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +from shlex import quote +from typing import Callable, Dict, List, Optional, Tuple, TYPE_CHECKING +from urllib.parse import urlsplit + +from napalm_device_types.models import ( + ServiceClusterDict, + ServiceClusterMemberDict, + ServiceClustersDict, +) +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "SVCC_BEGIN" +_END = "SVCC_END" +#: Exists only while systemd runs as PID 1; a container may carry systemctl without. +_SYSTEMD_RUN = "/run/systemd/system" +_PATRONI_CONFIG = "/etc/patroni/config.yml" +#: The container environment that may leave the host. +_ENV_KEYS = "ORIGIN_URL|REPLICA[0-9]*_URL|API_WHITELIST_IP|BUNKERWEB_INSTANCES|API_HTTP_PORT" +#: BunkerWeb's instance API, where a manager pushes its configuration. +_BUNKERWEB_API_PORT = 5000 + +#: Fed a unit's ``systemctl show`` output: ``active `` and ``config ``, +#: the first argument of the command line that is no flag. +_UNIT_AWK = " ".join( + ( + r'/^ActiveState=/ { print "active " substr($0, 13); next }', + r'/^ExecStart=/ { s = $0; i = index(s, "argv[]="); if (!i) next; s = substr(s, i + 7);' + r' j = index(s, " ;"); if (j) s = substr(s, 1, j - 1); n = split(s, t, " ");' + r' for (k = 2; k <= n; k++) if (!c && t[k] !~ /^-/) { print "config " t[k]; c = 1 } }', + ) +) + +#: Fed Patroni's YAML: ``scope``, ``name`` (top level only -- ``tags`` may carry a +#: ``name`` too), ``restapi`` (its connect_address) and ``tls yes`` when the REST +#: API has a certificate. Nothing else leaves the file. +_PATRONI_AWK = " ".join( + ( + r'function v(l) { sub(/^[^:]*:[ \t]*/, "", l); return l }', + r'/^[A-Za-z_][A-Za-z0-9_]*:/ { sec = $0; sub(/:.*/, "", sec);' + r' if (sec == "scope" || sec == "name") print sec " " v($0); next }', + r'sec == "restapi" && /^[ \t]+connect_address:/ { print "restapi " v($0) }', + r'sec == "restapi" && /^[ \t]+certfile:/ { print "tls yes" }', + ) +) + +#: Fed a container's environment, one ``KEY=value`` a line: the allowlisted keys, +#: each URL without its ``user:password@``. +_ENV_AWK = " ".join( + ( + r'{ x = index($0, "="); if (!x) next; k = substr($0, 1, x - 1); v = substr($0, x + 1);', + rf" if (k !~ /^({_ENV_KEYS})$/) next;", + r' gsub(/:\/\/[^@\/ ]*@/, "://", v); print "env " k " " v }', + ) +) + +_PATRONI_SH = ( + f"if command -v systemctl >/dev/null 2>&1 && [ -d {_SYSTEMD_RUN} ]; then echo '[systemd]'; " + "for u in $(systemctl list-units --all --plain --no-legend --type=service " + "patroni.service 'patroni@*' 2>/dev/null | awk '{print $1}'); do " + 'echo "[patroni $u]"; ' + 'o=$(systemctl show -p ActiveState -p ExecStart "$u" 2>/dev/null | ' + f"awk {quote(_UNIT_AWK)}); " + 'echo "$o"; ' + "f=$(echo \"$o\" | sed -n 's/^config //p' | head -n 1); " + f'[ -n "$f" ] || f={_PATRONI_CONFIG}; ' + 'if [ -r "$f" ]; then ' + f'p=$(awk {quote(_PATRONI_AWK)} "$f"); echo "$p"; ' + "r=$(echo \"$p\" | sed -n 's/^restapi //p' | head -n 1 | tr -d '\\042\\047 '); " + 's=http; case "$p" in *"tls yes"*) s=https;; esac; ' + 'if [ -n "$r" ]; then ' + 'b=$(curl -s -k -m 3 "$s://$r/cluster" 2>/dev/null || ' + 'wget -q -T 3 --no-check-certificate -O - "$s://$r/cluster" 2>/dev/null); ' + "if [ -n \"$b\" ]; then echo \"rest $(echo \"$b\" | tr -d '\\n')\"; " + "else echo rest-unreachable; fi; fi; " + 'elif [ -e "$f" ]; then echo "unreadable $f"; ' + 'else echo "missing $f"; fi; ' + "done; " + "else echo '[no-systemd]'; fi; " +) + + +def _docker_sh(binary: Optional[str]) -> str: + """The containers part: running containers of the clustered images, their + allowlisted environment. ``[no-docker]`` for a host without the engine.""" + if not binary: + return "echo '[no-docker]'; " + return ( + f"D={quote(binary)}; " + 'if command -v "$D" >/dev/null 2>&1; then ' + "if c=$(\"$D\" ps --format '{{.Names}}' 2>/dev/null); then echo '[docker]'; " + "for n in $c; do " + "i=$(\"$D\" inspect --format '{{.Config.Image}}' \"$n\" 2>/dev/null); " + 'case "$i" in *adguardhome-sync*|*bunkerweb*) ' + 'echo "[container $n]"; echo "image $i"; ' + "\"$D\" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"$n\" " + f"2>/dev/null | awk {quote(_ENV_AWK)};; esac; done; " + "else echo '[docker-unreadable]'; fi; " + "else echo '[no-docker]'; fi; " + ) + + +def service_clusters_command(docker: Optional[str] = "docker") -> str: + """One line of POSIX ``sh``, read-only. *docker* is the engine's binary as the + driver reaches it (QNAP's lives elsewhere), None for a host without one. The + frame markers are printed in two halves so that a transport which echoes the + command does not show them early.""" + return ( + "PATH=$PATH:/usr/sbin:/sbin; " + "printf '%s%s\\n' SVCC_ BEGIN; " + + _PATRONI_SH + + _docker_sh(docker) + + "printf '%s%s\\n' SVCC_ END" + ) + + +SERVICE_CLUSTERS_COMMAND = service_clusters_command() + +_COMMENT_RE = re.compile(r"\s+#.*$") +_Record = Tuple[str, str] +_Section = Tuple[str, str, List[_Record]] + + +def _frame(output: str) -> List[str]: + lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] + try: + start = lines.index(_BEGIN) + end = lines.index(_END, start) + except ValueError: + raise ValueError("no intact service cluster report in the output") from None + return lines[start + 1 : end] + + +def _sections(lines: List[str]) -> List[_Section]: + """``[ ]`` headers, each with its records as ``(kind, rest)``.""" + sections: List[_Section] = [] + for line in lines: + if line.startswith("[") and line.endswith("]"): + kind, _, name = line[1:-1].partition(" ") + sections.append((kind, name, [])) + elif line and sections: + kind, _, rest = line.partition(" ") + sections[-1][2].append((kind, rest.strip())) + return sections + + +def _clean(value: str) -> str: + """Without a trailing ``# comment`` and the quotes around it.""" + value = _COMMENT_RE.sub("", value.strip()) + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + return value.strip() + + +def _member( + *, + address: Optional[str] = None, + port: Optional[int] = None, + name: Optional[str] = None, + role: Optional[str] = None, + state: Optional[str] = None, + this_host: bool = False, + config_from: Optional[int] = None, +) -> ServiceClusterMemberDict: + return { + "name": name, + "address": address, + "port": port, + "role": role, + "state": state, + "this_host": this_host, + "config_from": config_from, + } + + +def _cluster( + mechanism: str, + service: str, + source: str, + cluster: Optional[str], + members: List[ServiceClusterMemberDict], + running: Optional[bool] = True, +) -> ServiceClusterDict: + return { + "mechanism": mechanism, + "service": service, + "source": source, + "cluster": cluster, + "running": running, + "members": members, + } + + +def _host_port(text: str) -> Tuple[Optional[str], Optional[int]]: + """``"10.0.0.5:8008"`` or ``"[fd00::5]:8008"`` -> host and port.""" + try: + split = urlsplit(f"//{text.strip()}") + return split.hostname, split.port + except ValueError: + return None, None + + +def _url(text: str) -> Tuple[Optional[str], Optional[int]]: + """A URL's host and port, the scheme's default where it names none.""" + value = text.strip() + if "://" not in value: + value = f"http://{value}" + try: + split = urlsplit(value) + port = split.port or {"http": 80, "https": 443}.get(split.scheme) + except ValueError: + return None, None + return split.hostname, port + + +# --- Patroni ------------------------------------------------------------------ + + +def _patroni_members(rest: str, own: Optional[str]) -> Optional[List[ServiceClusterMemberDict]]: + try: + members = json.loads(rest).get("members") + except (ValueError, AttributeError): + return None + if not isinstance(members, list): + return None + found = [] + for entry in members: + if not isinstance(entry, dict) or not entry.get("name"): + continue + port = entry.get("port") + found.append( + _member( + name=str(entry["name"]), + address=str(entry["host"]) if entry.get("host") else None, + port=port if isinstance(port, int) else None, + role=str(entry["role"]) if entry.get("role") else None, + state=str(entry["state"]) if entry.get("state") else None, + this_host=entry["name"] == own, + ) + ) + return sorted(found, key=lambda m: m["name"] or "") + + +def _patroni(unit: str, records: List[_Record]) -> List[ServiceClusterDict]: + values: Dict[str, str] = {} + rest: Optional[str] = None + for kind, text in records: + if kind in ("active", "scope", "name", "restapi") and kind not in values: + values[kind] = _clean(text) + elif kind == "rest": + rest = text + scope = values.get("scope") + if not scope: + return [] + own = values.get("name") or None + members = _patroni_members(rest, own) if rest else None + if not members: + address, _ = _host_port(values.get("restapi", "")) + members = [_member(name=own, address=address, this_host=True)] + running = values["active"] == "active" if "active" in values else None + source = unit[: -len(".service")] if unit.endswith(".service") else unit + return [_cluster("patroni", "patroni", source, scope, members, running)] + + +# --- Containers ---------------------------------------------------------------- + + +def _env(records: List[_Record]) -> Dict[str, str]: + env: Dict[str, str] = {} + for kind, text in records: + if kind == "env": + key, _, value = text.partition(" ") + env[key] = value.strip() + return env + + +def _image_name(records: List[_Record]) -> str: + """``bunkerity/bunkerweb@sha256:...`` -> ``bunkerweb``.""" + image = next((text for kind, text in records if kind == "image"), "") + return image.split("@")[0].rsplit("/", 1)[-1].split(":")[0] + + +def _adguardhome_sync(name: str, records: List[_Record]) -> Optional[List[ServiceClusterDict]]: + env = _env(records) + origin = _url(env["ORIGIN_URL"]) if env.get("ORIGIN_URL") else (None, None) + if origin[0] is None: + return None # configured by its YAML file, which is not read + replicas = sorted( + (key for key in env if re.fullmatch(r"REPLICA[0-9]*_URL", key)), + key=lambda key: int(key[7:-4] or 0), + ) + members = [_member(address=origin[0], port=origin[1], role="origin")] + for key in replicas: + host, port = _url(env[key]) + if host: + members.append(_member(address=host, port=port, role="replica", config_from=0)) + cluster = f"{origin[0]}:{origin[1]}" if origin[1] else origin[0] + return [_cluster("adguardhome-sync", "adguardhome", name, cluster, members)] + + +def _peer_address(entry: str) -> Optional[str]: + """An entry of a whitelist that names one peer: a single address that is no + loopback, link-local or unspecified one -- not a network.""" + text = entry.strip() + if "/" in text: + try: + network = ipaddress.ip_network(text, strict=False) + except ValueError: + return None + if network.num_addresses != 1: + return None + text = str(network.network_address) + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if address.is_loopback or address.is_link_local or address.is_unspecified: + return None + return str(address) + + +def _api_port(env: Dict[str, str]) -> int: + port = env.get("API_HTTP_PORT", "") + return int(port) if port.isdigit() else _BUNKERWEB_API_PORT + + +def _bunkerweb_worker(name: str, env: Dict[str, str]) -> List[ServiceClusterDict]: + managers = [ + address + for entry in re.split(r"[\s,]+", env.get("API_WHITELIST_IP", "")) + if entry and (address := _peer_address(entry)) + ] + if not managers: + return [] + members = [_member(port=_api_port(env), role="worker", this_host=True, config_from=1)] + members += [_member(address=address, role="manager") for address in managers] + return [_cluster("bunkerweb", "bunkerweb", name, managers[0], members)] + + +def _bunkerweb_manager(name: str, env: Dict[str, str]) -> List[ServiceClusterDict]: + instances = [entry for entry in env.get("BUNKERWEB_INSTANCES", "").split() if entry] + if not instances: + return [] + members = [_member(role="manager", this_host=True)] + for entry in instances: + host, port = _host_port(entry) + if host: + members.append( + _member(address=host, port=port or _BUNKERWEB_API_PORT, role="worker", config_from=0) + ) + return [_cluster("bunkerweb", "bunkerweb", name, None, members)] if len(members) > 1 else [] + + +def _bunkerweb(name: str, records: List[_Record]) -> List[ServiceClusterDict]: + image = _image_name(records) + if image == "bunkerweb": + return _bunkerweb_worker(name, _env(records)) + if image in ("bunkerweb-all-in-one", "bunkerweb-scheduler"): + return _bunkerweb_manager(name, _env(records)) + return [] # the UI, the API, autoconf: no instance and no manager + + +_CONTAINER_READERS: Dict[str, Callable[[str, List[_Record]], Optional[List[ServiceClusterDict]]]] = { + "adguardhome_sync": _adguardhome_sync, + "bunkerweb": _bunkerweb, +} + + +def _reader_for(records: List[_Record]) -> Optional[str]: + image = next((text for kind, text in records if kind == "image"), "") + if "adguardhome-sync" in image: + return "adguardhome_sync" + if "bunkerweb" in image: + return "bunkerweb" + return None + + +def _containers(sections: List[_Section]) -> Dict[str, Optional[List[ServiceClusterDict]]]: + kinds = {kind for kind, _, _ in sections} + if "no-docker" in kinds: + return {"adguardhome_sync": [], "bunkerweb": []} + if "docker" not in kinds: + return {"adguardhome_sync": None, "bunkerweb": None} + found: Dict[str, Optional[List[ServiceClusterDict]]] = {"adguardhome_sync": [], "bunkerweb": []} + for kind, name, records in sections: + key = _reader_for(records) if kind == "container" else None + if key is None or found[key] is None: + continue + clusters = _CONTAINER_READERS[key](name, records) + if clusters is None: + found[key] = None + else: + found[key].extend(clusters) # type: ignore[union-attr] + return found + + +def parse_service_clusters(output: str) -> ServiceClustersDict: + """Parse what :func:`service_clusters_command` printed. + + :raises ValueError: when the output carries no intact report. + """ + sections = _sections(_frame(output)) + patroni: Optional[List[ServiceClusterDict]] + if any(kind == "no-systemd" for kind, _, _ in sections): + patroni = None + elif any(kind == "patroni" and any(r[0] == "unreadable" for r in records) for kind, _, records in sections): + patroni = None + else: + patroni = [ + c for kind, unit, records in sections if kind == "patroni" for c in _patroni(unit, records) + ] + containers = _containers(sections) + return { + "patroni": patroni, + "adguardhome_sync": containers["adguardhome_sync"], + "bunkerweb": containers["bunkerweb"], + } + + +class ServiceClustersMixin: + """Adds :meth:`get_service_clusters` to a driver that can run a command on a Linux host. + + The template form (README, "Function classes"): the command and its parse are + the same everywhere, so they are concrete here, and a driver supplies only + :meth:`_run_service_clusters_command` -- how a command reaches its host, and + how it gains root there. A driver with a container engine of its own path + (``ContainerEngineMixin._container_engine_binary``) has it asked with that. + Mixed in by the drivers that can, so ``hasattr(driver, + "get_service_clusters")`` stays a truthful answer. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_service_clusters_command(self, command: str, *, privileged: bool) -> str: + """Run *command* on the host and return what it printed; as root when + *privileged*. The command is a single ``sh -c`` invocation of a few kB: + send it on an exec channel, not typed into a shell.""" + ... + + def _service_clusters_docker(self) -> str: + binary = getattr(self, "_container_engine_binary", None) + return binary("docker") if callable(binary) else "docker" + + def get_service_clusters(self) -> ServiceClustersDict: + """ + Returns the clusters the host's configuration says it belongs to, per + mechanism: Patroni (its scope and the members its DCS lists), + adguardhome-sync (its origin and replicas), BunkerWeb (a worker's manager, + a manager's instances). See :class:`~napalm_device_types.models.ServiceClusterDict`. + + None for a mechanism means it could not be read; ``[]`` that the host has none. + + Example:: + + { + "patroni": [ + {"mechanism": "patroni", "service": "patroni", "source": "patroni", + "cluster": "netork-db", + "running": True, + "members": [ + {"name": "db-01", "address": "10.7.234.12", "port": 5432, + "role": "leader", "state": "running", "this_host": True, + "config_from": None}, + {"name": "db-02", "address": "10.7.234.11", "port": 5432, + "role": "sync_standby", "state": "streaming", "this_host": False, + "config_from": None}, + ]}, + ], + "adguardhome_sync": [], + "bunkerweb": [], + } + + :raises ValueError: if neither reading carried an intact report. + """ + command = f"sh -c {quote(service_clusters_command(self._service_clusters_docker()))}" + try: + return parse_service_clusters(self._run_service_clusters_command(command, privileged=True)) + except ValueError: + pass + return parse_service_clusters(self._run_service_clusters_command(command, privileged=False)) diff --git a/pyproject.toml b/pyproject.toml index 25af5dc..84aa952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.16.0" +version = "4.17.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_service_clusters.py b/tests/test_service_clusters.py new file mode 100644 index 0000000..2bf8a7f --- /dev/null +++ b/tests/test_service_clusters.py @@ -0,0 +1,491 @@ +"""get_service_clusters: the configuration that says which services run together. + +Two hosts that answer at one virtual IP may run the same service by accident. +What proves a cluster is the configuration the members share: Patroni's scope +and the members its DCS lists, adguardhome-sync's origin and replicas, the +address a BunkerWeb worker takes its configuration from. Reading that is the +same on every Linux host, so the command and its parse live here once, and a +driver only carries the command across (netOrk #1125). + +The same configuration holds secrets -- Patroni's superuser password, the sync's +AdGuard logins, a database URI -- so the command filters on the host: only +allowlisted fields ever leave it. The tests below run the command for real to +hold it to that. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from napalm_device_types.service_clusters import ( + SERVICE_CLUSTERS_COMMAND, + ServiceClustersMixin, + parse_service_clusters, + service_clusters_command, +) + +SECRET = "s3cr3t-do-not-leak" + + +def _wire(*sections: str, systemd: bool = True, docker: str = "[docker]", noise: str = "") -> str: + head = "[systemd]" if systemd else "[no-systemd]" + body = "\n".join([head, *sections[:1], docker, *sections[1:]]) if sections else f"{head}\n{docker}" + return f"{noise}SVCC_BEGIN\n{body}\nSVCC_END\n" + + +DB01_CLUSTER = { + "members": [ + {"name": "db-01", "role": "leader", "state": "running", "host": "10.7.234.12", "port": 5432}, + {"name": "db-02", "role": "sync_standby", "state": "streaming", "host": "10.7.234.11", "port": 5432, "lag": 0}, + ], + "scope": "netork-db", +} + + +def _patroni(*records: str, unit: str = "patroni.service") -> str: + return "\n".join([f"[patroni {unit}]", *records]) + + +def _container(name: str, image: str, *env: str) -> str: + return "\n".join([f"[container {name}]", f"image {image}", *(f"env {e}" for e in env)]) + + +class TestPatroni: + def test_the_members_its_dcs_lists_with_their_roles(self): + reading = parse_service_clusters( + _wire( + _patroni( + "active active", + "config /etc/patroni/config.yml", + "scope netork-db", + "name db-01", + "restapi 10.7.234.12:8008", + "rest " + json.dumps(DB01_CLUSTER), + ) + ) + ) + + assert reading["patroni"] == [ + { + "mechanism": "patroni", + "service": "patroni", + "source": "patroni", + "cluster": "netork-db", + "running": True, + "members": [ + {"name": "db-01", "address": "10.7.234.12", "port": 5432, "role": "leader", + "state": "running", "this_host": True, "config_from": None}, + {"name": "db-02", "address": "10.7.234.11", "port": 5432, "role": "sync_standby", + "state": "streaming", "this_host": False, "config_from": None}, + ], + } + ] + + def test_without_its_rest_api_it_is_itself_in_its_scope(self): + reading = parse_service_clusters( + _wire( + _patroni( + "active active", "scope 'netork-db' # the cluster", "name db-01", + "restapi 10.7.234.12:8008", "rest-unreachable", + ) + ) + ) + + (cluster,) = reading["patroni"] + assert cluster["cluster"] == "netork-db" + assert cluster["members"] == [ + {"name": "db-01", "address": "10.7.234.12", "port": None, "role": None, + "state": None, "this_host": True, "config_from": None} + ] + + def test_an_unreadable_config_is_not_looked_at(self): + reading = parse_service_clusters( + _wire(_patroni("active active", "unreadable /etc/patroni/config.yml")) + ) + + assert reading["patroni"] is None + + def test_a_config_without_a_scope_declares_no_cluster(self): + assert parse_service_clusters(_wire(_patroni("active active", "name db-01")))["patroni"] == [] + + def test_a_host_without_systemd_is_not_looked_at(self): + assert parse_service_clusters(_wire(systemd=False))["patroni"] is None + + def test_no_patroni_is_looked_and_found_nothing(self): + assert parse_service_clusters(_wire())["patroni"] == [] + + +class TestAdguardhomeSync: + def test_the_origin_pushes_to_each_replica(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL http://10.9.100.5:3000", "REPLICA1_URL http://10.9.100.6:3000", + "REPLICA2_URL https://dns-03.example.org", + ), + ) + ) + + (cluster,) = reading["adguardhome_sync"] + assert (cluster["mechanism"], cluster["service"], cluster["cluster"]) == ( + "adguardhome-sync", "adguardhome", "10.9.100.5:3000", + ) + assert [(m["address"], m["port"], m["role"], m["config_from"]) for m in cluster["members"]] == [ + ("10.9.100.5", 3000, "origin", None), + ("10.9.100.6", 3000, "replica", 0), + ("dns-03.example.org", 443, "replica", 0), + ] + assert not any(m["this_host"] for m in cluster["members"]) + + def test_a_single_replica_url_counts(self): + reading = parse_service_clusters( + _wire("", _container("sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000", + "REPLICA_URL http://b:3000")) + ) + + assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == ["a", "b"] + + def test_a_sync_configured_by_file_is_not_looked_at(self): + """Its YAML is not read: the cluster is there, unseen.""" + reading = parse_service_clusters(_wire("", _container("sync", "bakito/adguardhome-sync"))) + + assert reading["adguardhome_sync"] is None + + +class TestBunkerweb: + def test_a_worker_takes_its_configuration_from_the_manager_it_admits(self): + reading = parse_service_clusters( + _wire("", _container("bunkerweb", "bunkerity/bunkerweb@sha256:2026", "API_WHITELIST_IP 10.50.100.5")) + ) + + (cluster,) = reading["bunkerweb"] + assert (cluster["service"], cluster["cluster"]) == ("bunkerweb", "10.50.100.5") + assert cluster["members"] == [ + {"name": None, "address": None, "port": 5000, "role": "worker", "state": None, + "this_host": True, "config_from": 1}, + {"name": None, "address": "10.50.100.5", "port": None, "role": "manager", "state": None, + "this_host": False, "config_from": None}, + ] + + def test_the_default_whitelist_names_no_manager(self): + """127.0.0.0/8 and a docker network are BunkerWeb's default, not a peer.""" + reading = parse_service_clusters( + _wire("", _container("bw", "bunkerity/bunkerweb:1.6", "API_WHITELIST_IP 127.0.0.0/8 10.20.30.0/24")) + ) + + assert reading["bunkerweb"] == [] + + def test_a_manager_names_its_instances(self): + reading = parse_service_clusters( + _wire("", _container("bw", "bunkerity/bunkerweb-all-in-one:1.6", + "BUNKERWEB_INSTANCES 10.50.100.6:5000 bw-03")), + ) + + (cluster,) = reading["bunkerweb"] + assert cluster["cluster"] is None # the manager cannot name its own address + assert [(m["address"], m["port"], m["role"], m["this_host"], m["config_from"]) + for m in cluster["members"]] == [ + (None, None, "manager", True, None), + ("10.50.100.6", 5000, "worker", False, 0), + ("bw-03", 5000, "worker", False, 0), + ] + + def test_a_manager_without_instances_is_no_cluster(self): + """proxy-01's all-in-one keeps its instances in its database.""" + reading = parse_service_clusters(_wire("", _container("bw", "bunkerity/bunkerweb-all-in-one@sha256:2b34"))) + + assert reading["bunkerweb"] == [] + + def test_its_ui_and_scheduler_side_containers_are_not_instances(self): + reading = parse_service_clusters( + _wire("", _container("ui", "bunkerity/bunkerweb-ui:1.6", "API_WHITELIST_IP 10.0.0.9")) + ) + + assert reading["bunkerweb"] == [] + + +class TestTheReport: + def test_a_host_without_docker_runs_no_containers(self): + reading = parse_service_clusters(_wire(docker="[no-docker]")) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], []) + + def test_a_docker_it_may_not_ask_is_not_looked_at(self): + reading = parse_service_clusters(_wire(docker="[docker-unreadable]")) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == (None, None) + + def test_noise_before_the_report_is_ignored(self): + assert parse_service_clusters(_wire(noise="Last login: today\n"))["patroni"] == [] + + def test_a_report_cut_short_raises_without_quoting_it(self): + with pytest.raises(ValueError) as caught: + parse_service_clusters("SVCC_BEGIN\n[systemd]\nscope netork-db\n") + + assert "netork-db" not in str(caught.value) + + +# --------------------------------------------------------------------------- +# The command itself, run against stubs and real files +# --------------------------------------------------------------------------- + +PATRONI_YML = f"""scope: netork-db +namespace: /service/ +name: db-01 + +restapi: + listen: 10.7.234.12:8008 + connect_address: 10.7.234.12:8008 + authentication: + username: patroni + password: {SECRET} + +etcd3: + hosts: + - 10.7.234.12:2379 + - 10.7.234.13:2379 + password: {SECRET} + +postgresql: + connect_address: 10.7.234.12:5432 + authentication: + superuser: + username: postgres + password: {SECRET} +tags: + name: not-the-member-name +""" + + +def _awks() -> list: + found = [] + for name, argv in (("mawk", ["mawk"]), ("busybox", ["busybox", "awk"]), ("gawk", ["gawk"])): + if shutil.which(argv[0]): + found.append(pytest.param(argv, id=name)) + return found or [pytest.param(None, marks=pytest.mark.skip(reason="no awk to run"))] + + +class _Host: + """A host as far as the command can tell: units, containers, a REST API, files.""" + + def __init__(self, root: Path, awk: list) -> None: + self.root = root + self.bin = root / "bin" + self.bin.mkdir() + self.state = root / "state" + self.state.mkdir() + self.booted = root / "run-systemd-system" + self.booted.mkdir() + (self.state / "units").write_text("") + (self.state / "containers").write_text("") + cat = shutil.which("cat") + self._stub( + "systemctl", + 'case "$1" in\n' + f' list-units) {cat} "$d/units" ;;\n' + f' show) for last; do :; done; {cat} "$d/show-$last" ;;\n' + "esac\n", + ) + self._stub( + "docker", + 'case "$1" in\n' + f' ps) {cat} "$d/containers" ;;\n' + ' inspect) for last; do :; done;\n' + ' case "$3" in *Config.Image*) ' + f'{cat} "$d/image-$last" ;; *) {cat} "$d/env-$last" ;; esac ;;\n' + "esac\n", + ) + self._stub("curl", f'for last; do :; done; {cat} "$d/rest" 2>/dev/null || exit 7\n') + for tool in ("sed", "head", "tr"): + if shutil.which(tool): + (self.bin / tool).symlink_to(shutil.which(tool)) + wrapper = self.bin / "awk" + program = " ".join([shutil.which(awk[0]) or awk[0], *awk[1:]]) + wrapper.write_text(f'#!/bin/sh\nexec {program} "$@"\n') + wrapper.chmod(0o755) + + def _stub(self, name: str, body: str) -> None: + stub = self.bin / name + stub.write_text(f"#!/bin/sh\nd={self.state}\n{body}") + stub.chmod(0o755) + + def unit(self, name: str, *properties: str) -> None: + with (self.state / "units").open("a") as units: + units.write(f"{name} loaded active running {name}\n") + (self.state / f"show-{name}").write_text("".join(f"{p}\n" for p in properties)) + + def container(self, name: str, image: str, *env: str) -> None: + with (self.state / "containers").open("a") as containers: + containers.write(f"{name}\n") + (self.state / f"image-{name}").write_text(f"{image}\n") + (self.state / f"env-{name}").write_text("".join(f"{e}\n" for e in env)) + + def rest(self, body: dict) -> None: + (self.state / "rest").write_text(json.dumps(body)) + + def file(self, relative: str, content: str) -> Path: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + def run(self, *, docker: bool = True) -> str: + command = service_clusters_command("docker" if docker else None) + assert command.count("/run/systemd/system") == 1 + command = command.replace("/run/systemd/system", str(self.booted)) + return subprocess.run( + ["/bin/sh", "-c", command], + capture_output=True, + text=True, + env={"PATH": str(self.bin)}, + timeout=30, + ).stdout + + +def _exec_start(*argv: str) -> str: + return f"ExecStart={{ path={argv[0]} ; argv[]={' '.join(argv)} ; ignore_errors=no ; pid=7 ; status=0/0 }}" + + +@pytest.fixture(params=_awks()) +def host(request, tmp_path) -> _Host: + return _Host(tmp_path, request.param) + + +class TestTheCommandOnAHost: + def test_patroni_as_on_db_01(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + host.rest(DB01_CLUSTER) + + out = host.run() + + assert SECRET not in out + assert "10.7.234.13" not in out # the DCS endpoints stay on the host + assert "not-the-member-name" not in out + (cluster,) = parse_service_clusters(out)["patroni"] + assert cluster["cluster"] == "netork-db" + assert [(m["name"], m["role"], m["this_host"]) for m in cluster["members"]] == [ + ("db-01", "leader", True), + ("db-02", "sync_standby", False), + ] + + def test_patroni_whose_rest_api_does_not_answer(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + + (cluster,) = parse_service_clusters(host.run())["patroni"] + + assert [(m["name"], m["address"], m["role"]) for m in cluster["members"]] == [ + ("db-01", "10.7.234.12", None) + ] + + @pytest.mark.skipif(os.geteuid() == 0, reason="root reads a file of mode 000") + def test_a_patroni_config_it_cannot_read_is_not_looked_at(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + config.chmod(0) + host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + + out = host.run() + + assert SECRET not in out + assert parse_service_clusters(out)["patroni"] is None + + def test_the_sync_and_a_worker_without_their_secrets(self, host): + host.container( + "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL=http://10.9.100.5:3000", f"ORIGIN_PASSWORD={SECRET}", + f"REPLICA1_URL=http://admin:{SECRET}@10.9.100.6:3000", f"REPLICA1_PASSWORD={SECRET}", + "CRON=*/5 * * * *", + ) + host.container( + "bunkerweb", "bunkerity/bunkerweb@sha256:2026", + "API_WHITELIST_IP=10.50.100.5", f"DATABASE_URI=mariadb+pymysql://bw:{SECRET}@db/bw", + ) + host.container("zabbix", "zabbix/zabbix-agent2:alpine-latest", f"ZBX_PSK={SECRET}") + + out = host.run() + + assert SECRET not in out + assert "admin" not in out + reading = parse_service_clusters(out) + assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == [ + "10.9.100.5", "10.9.100.6", + ] + assert reading["bunkerweb"][0]["cluster"] == "10.50.100.5" + assert reading["patroni"] == [] + + def test_a_host_without_docker(self, host): + reading = parse_service_clusters(host.run(docker=False)) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], []) + + +class TestTheMixin: + class _Driver(ServiceClustersMixin): + def __init__(self, answers: dict) -> None: + self.answers = answers + self.calls: list = [] + + def _run_service_clusters_command(self, command: str, *, privileged: bool) -> str: + self.calls.append(privileged) + return self.answers[privileged] + + def test_root_first(self): + driver = self._Driver({True: _wire(), False: "unused"}) + + assert driver.get_service_clusters()["patroni"] == [] + assert driver.calls == [True] + + def test_without_root_when_root_brought_no_report(self): + driver = self._Driver({True: "sudo: a password is required", False: _wire()}) + + assert driver.get_service_clusters()["bunkerweb"] == [] + assert driver.calls == [True, False] + + def test_the_container_engine_binary_is_the_drivers(self): + class _Qnap(self._Driver): + def _container_engine_binary(self, engine: str) -> str: + return "/share/CACHEDEV1_DATA/.qpkg/container-station/bin/docker" + + driver = _Qnap({True: _wire(), False: _wire()}) + seen: list = [] + + def run(command: str, *, privileged: bool) -> str: + seen.append(command) + return _wire() + + driver._run_service_clusters_command = run # type: ignore[method-assign] + driver.get_service_clusters() + + assert "container-station/bin/docker" in seen[0] + + def test_the_default_command_asks_docker(self): + assert "docker" in SERVICE_CLUSTERS_COMMAND + + +class TestTheSource: + """Where the evidence was read: the unit or container that carries it.""" + + def test_a_template_unit_is_named_after_its_instance(self): + reading = parse_service_clusters( + _wire(_patroni("active active", "scope pg", "name a", unit="patroni@16-main.service")) + ) + + assert reading["patroni"][0]["source"] == "patroni@16-main" + + def test_a_container_by_its_name(self): + reading = parse_service_clusters( + _wire("", _container("adguardhome-sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000"), + _container("bw", "bunkerity/bunkerweb", "API_WHITELIST_IP 10.0.0.5")) + ) + + assert reading["adguardhome_sync"][0]["source"] == "adguardhome-sync" + assert reading["bunkerweb"][0]["source"] == "bw" From 680500dd2ea343546034306afed2e2e2bd6d2928 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 18 Oct 2026 15:38:29 +0200 Subject: [PATCH 12/50] chore: release the service clusters as 4.18.0 feat/service-clusters (#47) and feat/opnsense-public-contracts (#46) both called themselves 4.17.0 and were merged one after the other: two trees under one version. 4.17.0 stays the public contracts (83cd7db, pinned by netOrk for #1113); the service clusters on top of it are 4.18.0. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 84aa952..552d896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.17.0" +version = "4.18.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" From 94bf399d34787a10697271b2407da03720621d71 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 18 Oct 2026 21:23:25 +0200 Subject: [PATCH 13/50] feat: ssh_host_keys.HostKeyCheck, trust on first use (NetOrk/netork#1167) Drivers get the host keys to expect in optional_args["ssh_host_keys"] (by port) and report what they saw through optional_args["ssh_host_key_observer"]. With a known key, any other key is refused before authentication (SshHostKeyMismatch); without one, the first key is accepted. One implementation for paramiko (HostKeyCheck.connect) and netmiko (HostKeyCheck.netmiko + observe). A caller or driver that ignores both keys keeps working as before. Version 4.19.0. --- README.md | 21 ++ napalm_device_types/__init__.py | 14 ++ napalm_device_types/ssh_host_keys.py | 300 +++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_ssh_host_keys.py | 275 ++++++++++++++++++++++++ 5 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/ssh_host_keys.py create mode 100644 tests/test_ssh_host_keys.py diff --git a/README.md b/README.md index c12f40d..e4367f6 100644 --- a/README.md +++ b/README.md @@ -254,6 +254,27 @@ NAPALM's own getters (`get_facts`, `get_interfaces`, `ping`, `get_config`) are n wrapped in a template — they belong to NAPALM, and code outside this repo relies on their contract. +### SSH host keys: trust on first use + +A driver that accepts any host key logs in with the device's credentials at whatever +answers on the device's address (NetOrk/netork#1167). The store of trusted keys is the +caller's; `ssh_host_keys.HostKeyCheck` is how every SSH driver verifies against it: + +- **In:** `optional_args["ssh_host_keys"]` = `{"22": {"key_type": "ssh-ed25519", "key": ""}}`, + by port. The driver takes the key for the port it connects to. With one, any other key is + refused *before* authentication and the connection raises `SshHostKeyMismatch` (a + `ConnectionException`). Without one, the first key is accepted. +- **Out:** `optional_args["ssh_host_key_observer"]`, a callable, gets one dict per + connection: `{"host", "port", "seen": {"key_type", "key", "fingerprint"}, "expected", "accepted"}`. + `accepted` is False only for a refused key. A failing observer is logged, never raised. +- **paramiko:** `HostKeyCheck(optional_args, host, port).connect(client, **connect_kwargs)`. +- **netmiko:** `with check.netmiko(self.netmiko_optional_args) as kwargs: ConnectHandler(..., **kwargs)`, + then `check.observe(conn.remote_conn_pre.get_transport())`. The known key reaches netmiko as a + `known_hosts` file that exists only inside the block, with `ssh_strict=True`. + +A caller that passes neither key gets first use on every connection, as before; a driver +that ignores both still connects. Old and new versions on either side keep working. + ## Design principle: generic vs. device-specific logic When adding behavior to a device-type base class, split it along one line: **would diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index c198e33..b5f0935 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -181,6 +181,14 @@ from napalm_device_types.ssh_server import ( SshHardeningResult, plan_ssh_hardening, ) +from napalm_device_types.ssh_host_keys import ( + KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, + OBSERVER_ARG as SSH_HOST_KEY_OBSERVER_ARG, + HostKeyCheck, + HostKeyObservation, + SshHostKey, + SshHostKeyMismatch, +) from napalm_device_types.storage import StorageDriver from napalm_device_types.switch import SwitchDriver @@ -288,6 +296,12 @@ __all__ = [ "SshHardeningPlan", "SshHardeningResult", "plan_ssh_hardening", + "SSH_HOST_KEYS_ARG", + "SSH_HOST_KEY_OBSERVER_ARG", + "HostKeyCheck", + "HostKeyObservation", + "SshHostKey", + "SshHostKeyMismatch", "parse_virtual_ips", "add_lag_interfaces", "driver_supports_ping", diff --git a/napalm_device_types/ssh_host_keys.py b/napalm_device_types/ssh_host_keys.py new file mode 100644 index 0000000..5c86e3f --- /dev/null +++ b/napalm_device_types/ssh_host_keys.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- +"""SSH host keys: trust on first use, and a changed key refused before any login. + +A driver that accepts whatever host key answers at a device's address logs in +there with the device's credentials, whoever that is (NetOrk/netork#1167). The +layer above keeps the store of the keys it trusts; a driver only verifies: + +* It receives the keys to expect in ``optional_args["ssh_host_keys"]`` + (:data:`KNOWN_KEYS_ARG`), by the port they were seen on: + ``{"22": {"key_type": "ssh-ed25519", "key": ""}}``, the two fields of + a ``known_hosts`` line. A driver takes the one for the port it connects to, so + the caller need not know which port that is (Proxmox: its API on 8006, SSH on + 22). With a key, any other key is refused before a password or a private key + leaves the caller. Without one, the first key is accepted: trust on first use. +* It reports every key it sees through ``optional_args["ssh_host_key_observer"]`` + (:data:`OBSERVER_ARG`), a callable that receives one dict per connection:: + + {"host": str, "port": int, + "seen": {"key_type", "key", "fingerprint"}, + "expected": {...} | None, + "accepted": bool} + + ``accepted`` is False only for a refused key; the connection then raises + :class:`SshHostKeyMismatch`. A failing observer is logged and ignored, so the + store being away never stops a connection that verified. + +A caller that passes neither keeps working as before (first use, every time), +and a driver that ignores both still connects for a caller that passes them: +old and new versions on either side do not break each other. + +:class:`HostKeyCheck` is the one implementation, for a paramiko ``SSHClient`` +(:meth:`~HostKeyCheck.connect`) and for netmiko (:meth:`~HostKeyCheck.netmiko`, +then :meth:`~HostKeyCheck.observe`). Like :mod:`napalm_device_types.channel` it +works on the objects paramiko hands it and imports no SSH library itself. +""" + +from __future__ import annotations + +import base64 +import hashlib +import logging +import os +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List, Mapping, Optional, Set + +from napalm.base.exceptions import ConnectionException + +logger = logging.getLogger(__name__) + +#: The ``optional_args`` key that carries the host keys to expect, by port. +KNOWN_KEYS_ARG = "ssh_host_keys" +#: The ``optional_args`` key that carries the callable every seen key goes to. +OBSERVER_ARG = "ssh_host_key_observer" + +#: The netmiko arguments this module decides; a caller's own values give way. +_NETMIKO_HOST_KEY_ARGS = ("ssh_strict", "system_host_keys", "alt_host_keys", "alt_key_file") + +_DEFAULT_SSH_PORT = 22 + + +@dataclass(frozen=True) +class SshHostKey: + """A host's public key, as the two fields of a ``known_hosts`` line.""" + + key_type: str + #: The public key blob, base64. + key: str + + @property + def fingerprint(self) -> str: + """``SHA256:…``, as ``ssh-keygen -l`` and the ssh client print it.""" + digest = hashlib.sha256(base64.b64decode(self.key)).digest() + return "SHA256:" + base64.b64encode(digest).decode("ascii").rstrip("=") + + def as_dict(self) -> Dict[str, str]: + return {"key_type": self.key_type, "key": self.key, "fingerprint": self.fingerprint} + + @classmethod + def from_value(cls, value: Any) -> Optional["SshHostKey"]: + """The key in *value* (a mapping with ``key_type`` and ``key``), or None for none.""" + if not value: + return None + if not isinstance(value, Mapping) or not value.get("key_type") or not value.get("key"): + raise ValueError(f"not an SSH host key: {value!r}") + return cls(str(value["key_type"]), str(value["key"])) + + @classmethod + def of(cls, pkey: Any) -> "SshHostKey": + """The key of a paramiko ``PKey`` (what a transport says the server presented).""" + return cls(pkey.get_name(), pkey.get_base64()) + + def known_hosts_line(self, host: str, port: int) -> str: + """The line under which an ssh client looks this host up.""" + name = host if port == _DEFAULT_SSH_PORT else f"[{host}]:{port}" + return f"{name} {self.key_type} {self.key}" + + +@dataclass(frozen=True) +class HostKeyObservation: + """The key one connection saw, against the key it was to expect.""" + + host: str + port: int + seen: SshHostKey + expected: Optional[SshHostKey] + + @property + def accepted(self) -> bool: + """True on first use and for the expected key; False for a refused one.""" + return self.expected is None or self.expected == self.seen + + def as_dict(self) -> Dict[str, Any]: + return { + "host": self.host, + "port": self.port, + "seen": self.seen.as_dict(), + "expected": self.expected.as_dict() if self.expected else None, + "accepted": self.accepted, + } + + +class SshHostKeyMismatch(ConnectionException): # type: ignore[misc] + """The device presented another host key than the one it is known by. + + Raised before any authentication: no password and no private key went to it. + """ + + def __init__(self, observation: HostKeyObservation) -> None: + self.observation = observation + expected = observation.expected + trusted = f"{expected.fingerprint} ({expected.key_type})" if expected else "none" + super().__init__( + f"SSH host key mismatch at {observation.host}:{observation.port}: it presented " + f"{observation.seen.fingerprint} ({observation.seen.key_type}), the trusted key is " + f"{trusted}. Nothing was sent to it." + ) + + +class _AcceptFirst: + """A paramiko missing-host-key policy for first use: accept, keep nothing. + + The key is reported once the connection stands (:meth:`HostKeyCheck.observe`); + nothing is written to a ``known_hosts`` file. + """ + + def missing_host_key(self, client: Any, hostname: str, key: Any) -> None: + return None + + +class _RefuseUnknown: + """A paramiko policy for a host with a known key: refuse anything else. + + paramiko compares a loaded key itself and raises ``BadHostKeyException`` + for another one; this only answers if the loaded key was not found under + the name connected to, which must not turn into an accepted key. + """ + + def missing_host_key(self, client: Any, hostname: str, key: Any) -> None: + raise ConnectionException(f"SSH host key of {hostname} is not the one expected") + + +def _expected_key(known: Any, port: int) -> Optional[SshHostKey]: + """The key in *known* (``{port: key}``, the port as text or number) for *port*.""" + if not known: + return None + if not isinstance(known, Mapping): + raise ValueError(f"SSH host keys must be given by port, not as {type(known).__name__}") + return SshHostKey.from_value(known.get(str(port)) or known.get(port)) + + +def _refused_key(exc: BaseException) -> Optional[Any]: + """paramiko's ``BadHostKeyException`` in *exc*'s chain (netmiko and drivers wrap it).""" + seen: Set[int] = set() + stack: List[Optional[BaseException]] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + if hasattr(current, "expected_key") and hasattr(current, "key"): + return current + stack.extend([current.__cause__, current.__context__]) + return None + + +class HostKeyCheck: + """How one driver connection verifies its host's key. + + *optional_args* are the driver's (both keys optional), *host* and *port* the + address the connection goes to, as given to paramiko or netmiko. + """ + + def __init__(self, optional_args: Optional[Mapping[str, Any]], host: str, port: int) -> None: + args = optional_args or {} + self.host = host + self.port = int(port) + self.expected = _expected_key(args.get(KNOWN_KEYS_ARG), self.port) + self._observer = args.get(OBSERVER_ARG) + + # ------------------------------------------------------------------ + # paramiko + # ------------------------------------------------------------------ + + def prepare(self, client: Any) -> None: + """Before ``client.connect()``: the known key, or acceptance of the first one. + + The known key goes into the client's host keys, so paramiko asks the + server for that key type first and refuses any other key itself. + """ + if self.expected is None: + client.set_missing_host_key_policy(_AcceptFirst()) + return + with self._known_hosts_file() as path: + client.get_host_keys().load(path) + client.set_missing_host_key_policy(_RefuseUnknown()) + + def connect(self, client: Any, **kwargs: Any) -> HostKeyObservation: + """``client.connect()`` to this host with *kwargs*, verified and reported.""" + self.prepare(client) + with self.refusing(): + client.connect(hostname=self.host, port=self.port, **kwargs) + return self.observe(client.get_transport()) + + # ------------------------------------------------------------------ + # netmiko + # ------------------------------------------------------------------ + + @contextmanager + def netmiko(self, base: Mapping[str, Any]) -> Iterator[Dict[str, Any]]: + """``ConnectHandler`` keyword arguments: *base* with the host key decided. + + Open the connection inside the block and call :meth:`observe` on its + transport after it. A known key goes to netmiko in a ``known_hosts`` + file that exists only for the block, with ``ssh_strict``: should the + file not be read, netmiko refuses rather than accepts. + """ + kwargs = {k: v for k, v in base.items() if k not in _NETMIKO_HOST_KEY_ARGS} + kwargs["system_host_keys"] = False + if self.expected is None: + kwargs.update(ssh_strict=False, alt_host_keys=False) + with self.refusing(): + yield kwargs + return + with self._known_hosts_file() as path: + kwargs.update(ssh_strict=True, alt_host_keys=True, alt_key_file=path) + with self.refusing(): + yield kwargs + + # ------------------------------------------------------------------ + # both + # ------------------------------------------------------------------ + + @contextmanager + def refusing(self) -> Iterator[None]: + """Around a connect: a refused host key becomes :class:`SshHostKeyMismatch`.""" + try: + yield + except Exception as exc: + refused = _refused_key(exc) + if refused is None: + raise + expected = self.expected or SshHostKey.of(refused.expected_key) + offered = SshHostKey.of(refused.key) + observation = HostKeyObservation(self.host, self.port, offered, expected) + self._report(observation) + raise SshHostKeyMismatch(observation) from exc + + def observe(self, transport: Any) -> HostKeyObservation: + """After a connect: the key the server presented on *transport*, reported.""" + seen = SshHostKey.of(transport.get_remote_server_key()) + observation = HostKeyObservation(self.host, self.port, seen, self.expected) + self._report(observation) + return observation + + def _report(self, observation: HostKeyObservation) -> None: + if not callable(self._observer): + return + try: + self._observer(observation.as_dict()) + except Exception: # the store is the caller's; a connection does not hang on it + logger.warning( + "Could not report the SSH host key of %s:%s", self.host, self.port, exc_info=True + ) + + @contextmanager + def _known_hosts_file(self) -> Iterator[str]: + """A private ``known_hosts`` file holding only the expected key.""" + assert self.expected is not None + fd, path = tempfile.mkstemp(prefix="known_hosts-") + try: + with os.fdopen(fd, "w") as f: + f.write(self.expected.known_hosts_line(self.host, self.port) + "\n") + yield path + finally: + try: + os.unlink(path) + except OSError: + pass diff --git a/pyproject.toml b/pyproject.toml index 552d896..39867e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.18.0" +version = "4.19.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_ssh_host_keys.py b/tests/test_ssh_host_keys.py new file mode 100644 index 0000000..0a56bee --- /dev/null +++ b/tests/test_ssh_host_keys.py @@ -0,0 +1,275 @@ +"""SSH host keys: trust on first use, and a changed key refused before any login. + +The layer above keeps the store (NetOrk/netork#1167). A driver gets the key it +should expect through ``optional_args["ssh_host_keys"]`` (by port) and reports every key it +sees through ``optional_args["ssh_host_key_observer"]``. These tests run real +paramiko against an in-process SSH server, because the property that matters — +nothing is sent to a host with the wrong key — is paramiko's behaviour once the +key is loaded, not something a fake could show. +""" + +from __future__ import annotations + +import os +import socket +import threading + +import paramiko +import pytest +from napalm.base.exceptions import ConnectionException + +from napalm_device_types.ssh_host_keys import ( + KNOWN_KEYS_ARG, + OBSERVER_ARG, + HostKeyCheck, + SshHostKey, + SshHostKeyMismatch, +) + +#: An ed25519 public key and the fingerprint ``ssh-keygen -lf`` prints for it. +ED25519 = SshHostKey( + "ssh-ed25519", "AAAAC3NzaC1lZDI1NTE5AAAAIJH2oO06vroJT2ooSB3/2tu8HufXQng4xQlxAk/4raee" +) +ED25519_FP = "SHA256:26dxfTuve20LGGTpJ1T0ns11/ID3XMhbOahQq3k75TU" + + +class _Server(paramiko.ServerInterface): + """Accepts any password and remembers each one it was offered.""" + + def __init__(self) -> None: + self.passwords: list[str] = [] + + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + self.passwords.append(password) + return paramiko.AUTH_SUCCESSFUL + + +class _Host: + """An SSH server on one end of a socket pair, with *host_key*.""" + + def __init__(self, host_key: paramiko.PKey) -> None: + self.client_sock, server_sock = socket.socketpair() + self.transport = paramiko.Transport(server_sock) + self.transport.add_server_key(host_key) + self.server = _Server() + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + try: + self.transport.start_server(server=self.server) + except Exception: # the client hung up on a refused key + pass + + def close(self) -> None: + self.transport.close() + self.client_sock.close() + self._thread.join(timeout=5) + + +@pytest.fixture +def host_key() -> paramiko.PKey: + return paramiko.ECDSAKey.generate() + + +@pytest.fixture +def host(host_key): + h = _Host(host_key) + yield h + h.close() + + +def _connect(check: HostKeyCheck, host: _Host) -> paramiko.SSHClient: + client = paramiko.SSHClient() + check.connect( + client, + username="admin", + password="s3cret", + sock=host.client_sock, + look_for_keys=False, + allow_agent=False, + timeout=5, + ) + return client + + +def _check(expected: SshHostKey | None, seen: list, port: int = 22) -> HostKeyCheck: + args: dict = {OBSERVER_ARG: seen.append} + if expected is not None: + args[KNOWN_KEYS_ARG] = {str(port): expected.as_dict()} + return HostKeyCheck(args, "10.0.0.5", port) + + +class TestSshHostKey: + def test_fingerprint_is_the_one_ssh_keygen_prints(self): + assert ED25519.fingerprint == ED25519_FP + + def test_of_reads_a_paramiko_key(self, host_key): + key = SshHostKey.of(host_key) + assert key.key_type == host_key.get_name() + assert key.key == host_key.get_base64() + + def test_as_dict_round_trips(self): + d = ED25519.as_dict() + assert d == {"key_type": "ssh-ed25519", "key": ED25519.key, "fingerprint": ED25519_FP} + assert SshHostKey.from_value(d) == ED25519 + + @pytest.mark.parametrize("value", [None, {}, ""]) + def test_no_value_is_no_key(self, value): + assert SshHostKey.from_value(value) is None + + @pytest.mark.parametrize( + "value", [{"key_type": "ssh-ed25519"}, {"key": "AAAA"}, "ssh-ed25519 AAAA"] + ) + def test_a_malformed_value_is_refused(self, value): + with pytest.raises(ValueError): + SshHostKey.from_value(value) + + def test_known_hosts_line_names_a_port_other_than_22(self): + assert ED25519.known_hosts_line("10.0.0.5", 22) == f"10.0.0.5 ssh-ed25519 {ED25519.key}" + line = ED25519.known_hosts_line("10.0.0.5", 2222) + assert line == f"[10.0.0.5]:2222 ssh-ed25519 {ED25519.key}" + + +class TestKnownKeys: + def test_the_key_is_the_one_for_the_port_connected_to(self): + args = {KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}} + assert HostKeyCheck(args, "10.0.0.5", 22).expected == ED25519 + assert HostKeyCheck(args, "10.0.0.5", 2222).expected is None + + def test_a_port_may_be_given_as_a_number(self): + args = {KNOWN_KEYS_ARG: {2222: ED25519.as_dict()}} + assert HostKeyCheck(args, "10.0.0.5", 2222).expected == ED25519 + + def test_a_malformed_mapping_is_refused(self): + with pytest.raises(ValueError): + HostKeyCheck({KNOWN_KEYS_ARG: [ED25519.as_dict()]}, "10.0.0.5", 22) + + +class TestParamiko: + def test_first_use_accepts_the_key_and_reports_it(self, host, host_key): + seen: list = [] + _connect(_check(None, seen), host).close() + assert host.server.passwords == ["s3cret"] + assert seen == [ + { + "host": "10.0.0.5", + "port": 22, + "seen": SshHostKey.of(host_key).as_dict(), + "expected": None, + "accepted": True, + } + ] + + def test_the_known_key_connects(self, host, host_key): + seen: list = [] + known = SshHostKey.of(host_key) + _connect(_check(known, seen), host).close() + assert host.server.passwords == ["s3cret"] + assert seen[0]["accepted"] is True + assert seen[0]["expected"] == known.as_dict() + + def test_a_changed_key_is_refused_before_any_login(self, host, host_key): + seen: list = [] + other = SshHostKey.of(paramiko.ECDSAKey.generate()) + with pytest.raises(SshHostKeyMismatch) as caught: + _connect(_check(other, seen), host) + assert host.server.passwords == [] + assert caught.value.observation.seen == SshHostKey.of(host_key) + assert caught.value.observation.expected == other + assert seen[0]["accepted"] is False + assert seen[0]["seen"]["fingerprint"] == SshHostKey.of(host_key).fingerprint + + def test_a_key_of_another_type_is_refused(self, host): + seen: list = [] + with pytest.raises(SshHostKeyMismatch): + _connect(_check(ED25519, seen), host) + assert host.server.passwords == [] + + def test_the_mismatch_is_a_connection_failure_that_names_both_keys(self, host, host_key): + with pytest.raises(ConnectionException) as caught: + _connect(_check(ED25519, []), host) + message = str(caught.value) + assert "10.0.0.5:22" in message + assert SshHostKey.of(host_key).fingerprint in message + assert ED25519_FP in message + + def test_a_failing_observer_does_not_break_the_connection(self, host): + def boom(_observation): + raise RuntimeError("store down") + + check = HostKeyCheck({OBSERVER_ARG: boom}, "10.0.0.5", 22) + _connect(check, host).close() + assert host.server.passwords == ["s3cret"] + + def test_without_optional_args_it_still_connects(self, host): + _connect(HostKeyCheck(None, "10.0.0.5", 22), host).close() + assert host.server.passwords == ["s3cret"] + + +class TestNetmiko: + BASE = { + "ssh_strict": False, + "alt_host_keys": True, + "alt_key_file": "/x", + "system_host_keys": True, + "conn_timeout": 9, + } + + def test_first_use_lets_netmiko_accept_the_key(self): + with HostKeyCheck({}, "10.0.0.5", 22).netmiko(self.BASE) as kwargs: + assert kwargs == { + "ssh_strict": False, + "alt_host_keys": False, + "system_host_keys": False, + "conn_timeout": 9, + } + + def test_a_known_key_is_the_only_one_netmiko_takes(self): + check = HostKeyCheck({KNOWN_KEYS_ARG: {"2222": ED25519.as_dict()}}, "10.0.0.5", 2222) + with check.netmiko(self.BASE) as kwargs: + path = kwargs["alt_key_file"] + assert kwargs["ssh_strict"] is True + assert kwargs["alt_host_keys"] is True + assert kwargs["system_host_keys"] is False + assert kwargs["conn_timeout"] == 9 + with open(path) as f: + assert f.read().strip() == ED25519.known_hosts_line("10.0.0.5", 2222) + assert not os.path.exists(path) + + def test_a_refusal_inside_netmiko_becomes_the_mismatch(self, host_key): + # netmiko re-raises paramiko's BadHostKeyException as its own exception, + # inside the except block, and a driver wraps that once more. + seen: list = [] + check = _check(ED25519, seen) + with pytest.raises(SshHostKeyMismatch) as caught: + with check.netmiko({}): + try: + try: + other = paramiko.ECDSAKey.generate() + raise paramiko.BadHostKeyException("10.0.0.5", host_key, other) + except paramiko.SSHException as exc: + raise RuntimeError(f"A paramiko SSHException occurred: {exc}") + except RuntimeError as exc: + raise ConnectionException(str(exc)) from exc + assert caught.value.observation.seen == SshHostKey.of(host_key) + assert caught.value.observation.expected == ED25519 + assert seen[0]["accepted"] is False + + def test_any_other_failure_passes_unchanged(self): + with pytest.raises(ConnectionException, match="timed out"): + with HostKeyCheck({KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}}, "10.0.0.5", 22).netmiko({}): + raise ConnectionException("timed out") + + def test_observe_reports_the_key_netmiko_saw(self, host_key): + class _Transport: + def get_remote_server_key(self): + return host_key + + seen: list = [] + observation = _check(None, seen).observe(_Transport()) + assert observation.accepted is True + assert seen[0]["seen"] == SshHostKey.of(host_key).as_dict() From cce26eff441921678125c0e21ad62512b49dcc7b Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 19 Oct 2026 17:38:57 +0200 Subject: [PATCH 14/50] feat: ServicePackagesMixin, which packages a systemd service is (NetOrk/netork#1236) get_service_packages() returns, per systemd service, the installed packages it is made of: the one shipping its unit file and the one shipping the binary its ExecStart runs -- not the libraries that binary links against. One read-only command: systemctl show for every loaded service unit, then one dpkg -S for all paths (each also across the /usr merge), or rpm -qf per path. The report is framed; one cut short raises. A driver supplies _run_service_packages_command, as for KernelFactsMixin. Version 4.20.0. --- README.md | 7 + napalm_device_types/__init__.py | 9 ++ napalm_device_types/service_packages.py | 182 ++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_service_packages.py | 150 +++++++++++++++++++ 5 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/service_packages.py create mode 100644 tests/test_service_packages.py diff --git a/README.md b/README.md index e4367f6..0b2ba42 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,13 @@ everywhere, so the command and its parse are concrete here and a driver supplies `_run_kernel_facts_command`. `OSDriver` does not carry it — a Windows host is an OS driver too, and `hasattr(driver, "get_kernel_facts")` has to stay truthful. +`ServicePackagesMixin` (`get_service_packages`) is mixed in the same way: which installed +packages each systemd service is made of — the one shipping its unit file and the one +shipping the binary its `ExecStart` runs, by `dpkg -S` or `rpm -qf`, not the libraries +that binary links against. An application needs it to let a statement about a service +(the postgresql unit) reach the findings on its packages. A driver supplies +`_run_service_packages_command`. + `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer kernel of the running flavour installed), whether it patches itself (unattended-upgrades, diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index b5f0935..e9a3bf1 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -52,6 +52,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.packages.PackageManagementMixin` * :class:`~napalm_device_types.ping_sweep.PingSweepMixin` * :class:`~napalm_device_types.service_clusters.ServiceClustersMixin` +* :class:`~napalm_device_types.service_packages.ServicePackagesMixin` * :class:`~napalm_device_types.services.ServiceControlMixin` * :class:`~napalm_device_types.systemd.SystemdServicesMixin` * :class:`~napalm_device_types.updates.UpdateMixin` @@ -98,6 +99,11 @@ from napalm_device_types.licenses import LicenseMixin from napalm_device_types.host_reboot import HostRebootMixin from napalm_device_types.interface_filter import InterfaceFilterMixin from napalm_device_types.kernel import KERNEL_FACTS_COMMAND, KernelFactsMixin, parse_kernel_facts +from napalm_device_types.service_packages import ( + SERVICE_PACKAGES_COMMAND, + ServicePackagesMixin, + parse_service_packages, +) from napalm_device_types.lag import add_lag_interfaces from napalm_device_types.listening import ( LISTENING_SOCKETS_COMMAND, @@ -265,6 +271,9 @@ __all__ = [ "KernelFactsMixin", "KERNEL_FACTS_COMMAND", "parse_kernel_facts", + "SERVICE_PACKAGES_COMMAND", + "ServicePackagesMixin", + "parse_service_packages", "LISTENING_SOCKETS_COMMAND", "ListeningSocketsMixin", "ListeningSocketsUnavailable", diff --git a/napalm_device_types/service_packages.py b/napalm_device_types/service_packages.py new file mode 100644 index 0000000..c0dc146 --- /dev/null +++ b/napalm_device_types/service_packages.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +"""Which installed packages a systemd service is made of. + +A vulnerability is found on a *package*; a statement about a *service* -- the +postgresql unit, nginx -- reaches its findings only once it is known which +packages that service is. Here that is the package shipping the unit file and +the one shipping the binary its ``ExecStart`` runs: ``nginx-common`` and +``nginx-core``, ``postgresql-common`` and ``postgresql-16``. Reading it is the +same on every systemd host, so the command and its parse live here once and a +driver only carries the command across. + +**Only the unit's own packages.** The libraries its binary links against +(``libssl3`` under nginx) are every service's at once; a statement about nginx +is not one about libc, and following the dependencies would make it one. + +The command is read-only and needs no privileges. It prints ``systemctl show`` +for every loaded service unit and then asks the package manager about every +path at once: ``dpkg -S`` once for all of them -- one call, since each loads the +whole file list, and per path that is minutes on an ordinary host -- or +``rpm -qf`` per path. dpkg knows a file by the path its package shipped, so each +path is asked across the ``/usr`` merge too. The report is framed; one whose end +is missing raises, because a list cut short must never read as services that own +nothing. +""" + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Set, Tuple, TYPE_CHECKING + +_BEGIN = "SVCPKG_BEGIN" +_END = "SVCPKG_END" +_SUFFIX = ".service" + +#: A Debian or RPM package name. +_PACKAGE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9+._-]*$") +#: The binary of an ``ExecStart`` as ``systemctl show`` prints it. +_EXEC_PATH = re.compile(r"\bpath=(/[^ ;]+)") + +#: Every unit file and ``ExecStart`` binary in ``$s``, each also across the +#: ``/usr`` merge, one per line. +_PATHS = ( + "printf '%s\\n' \"$s\" | sed -n -e 's/^FragmentPath=\\(\\/.*\\)/\\1/p' " + "-e 's/^ExecStart=.*path=\\(\\/[^ ;]*\\).*/\\1/p' | sort -u " + "| sed -e 'p' -e 's#^/usr/#/#;t' -e 's#^/#/usr/#'" +) + +#: One line, POSIX ``sh``, read-only. The frame markers are printed in two +#: halves so that a transport which echoes the command does not show them early. +SERVICE_PACKAGES_COMMAND = ( + "s=$(systemctl list-units --type=service --all --no-legend --no-pager --plain 2>/dev/null " + "| awk '{print $1}' | tr '\\n' '\\0' | xargs -0 -r systemctl show --no-pager " + "-p Id -p FragmentPath -p ExecStart -- 2>/dev/null); " + "printf '%s%s\\n' SVCPKG_ BEGIN; echo '[units]'; printf '%s\\n\\n' \"$s\"; " + "if command -v dpkg >/dev/null 2>&1; then echo '[dpkg]'; " + f"{_PATHS} | tr '\\n' '\\0' | xargs -0 -r dpkg -S 2>/dev/null; " + "elif command -v rpm >/dev/null 2>&1; then echo '[rpm]'; " + f"{_PATHS} | while IFS= read -r f; do " + 'printf \'%s\\t%s\\n\' "$f" "$(rpm -qf --qf \'%{NAME} \' "$f" 2>/dev/null)"; done; fi; ' + "printf '%s%s\\n' SVCPKG_ END" +) + + +def _report(output: str) -> Dict[str, List[str]]: + lines = [line.rstrip() for line in output.splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact service package report in the output") from None + sections: Dict[str, List[str]] = {} + current: Optional[List[str]] = None + for line in lines[start + 1 : end]: + if line in ("[units]", "[dpkg]", "[rpm]"): + current = sections.setdefault(line[1:-1], []) + elif current is not None: + current.append(line) + return sections + + +def _units(lines: List[str]) -> Dict[str, Set[str]]: + """``systemctl show`` blocks -> ``{unit: its unit file and binary}``.""" + units: Dict[str, Set[str]] = {} + unit, paths = "", set() + for line in [*lines, ""]: + key, _, value = line.partition("=") + if not line.strip(): + if unit: + units[unit] = paths + unit, paths = "", set() + elif key == "Id" and value.endswith(_SUFFIX): + unit = value[: -len(_SUFFIX)] + elif key == "FragmentPath" and value.startswith("/"): + paths.add(value) + elif key == "ExecStart": + match = _EXEC_PATH.search(value) + if match: + paths.add(match.group(1)) + return units + + +def _names(raw: str) -> Set[str]: + """``libfoo1:amd64, foo-bin`` -> ``{"libfoo1", "foo-bin"}``; nothing else counts.""" + found = set() + for name in raw.replace(",", " ").split(): + name = name.split(":", 1)[0] + if _PACKAGE.match(name): + found.add(name) + return found + + +def _owners(sections: Dict[str, List[str]]) -> Dict[str, Set[str]]: + """``{path: the packages owning it}`` from dpkg's or rpm's answer.""" + owners: Dict[str, Set[str]] = {} + for line in sections.get("dpkg", []): + # "pkg[, pkg2][:arch]: /path"; "diversion by x from: /path" is no owner. + if line.startswith("diversion by") or ": /" not in line: + continue + names, _, path = line.partition(": /") + owners.setdefault("/" + path.strip(), set()).update(_names(names)) + for line in sections.get("rpm", []): + path, _, names = line.partition("\t") + if " is not owned" not in names: + owners.setdefault(path.strip(), set()).update(_names(names)) + return owners + + +def _across_usr_merge(path: str) -> Tuple[str, str]: + return (path, path[4:] if path.startswith("/usr/") else "/usr" + path) + + +def parse_service_packages(output: str) -> Dict[str, List[str]]: + """Parse what :data:`SERVICE_PACKAGES_COMMAND` printed. + + ``{unit name without ".service": its packages, sorted}``. A unit no package + owns -- a unit file an admin wrote -- maps to an empty list: read, and none. + + :raises ValueError: when the output carries no intact report. + """ + sections = _report(output) + owners = _owners(sections) + return { + unit: sorted( + {name for p in paths for alt in _across_usr_merge(p) for name in owners.get(alt, ())} + ) + for unit, paths in _units(sections.get("units", [])).items() + } + + +class ServicePackagesMixin: + """Adds :meth:`get_service_packages` to a driver that can run a command on a systemd host. + + The template form, like :class:`~napalm_device_types.kernel.KernelFactsMixin`: + the reading and its parse are concrete here, and a driver supplies only + :meth:`_run_service_packages_command` -- how a command reaches its host. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_service_packages_command(self, command: str) -> str: + """Run *command* on the host with ``sh`` and return what it printed.""" + ... + + def get_service_packages(self) -> Dict[str, List[str]]: + """ + Returns, per systemd service, the installed packages it is made of. + + The package that ships its unit file and the one that ships the binary + its ``ExecStart`` runs -- not the libraries that binary links against. + + Example:: + + { + "nginx": ["nginx-common", "nginx-core"], + "postgresql@16-main": ["postgresql-16", "postgresql-common"], + "local-backup": [], + } + + :raises ValueError: if the host's output carried no intact report. + """ + return parse_service_packages(self._run_service_packages_command(SERVICE_PACKAGES_COMMAND)) diff --git a/pyproject.toml b/pyproject.toml index 39867e8..7bb03aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.19.0" +version = "4.20.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_service_packages.py b/tests/test_service_packages.py new file mode 100644 index 0000000..0a1a45b --- /dev/null +++ b/tests/test_service_packages.py @@ -0,0 +1,150 @@ +"""get_service_packages: which installed packages a systemd service is (NetOrk/netork#1236). + +A vulnerability is found on a *package*; a decision about a *service* -- the +postgresql unit, nginx -- means something only once it is known which packages +that service is. The package that ships its unit file and the one that ships the +binary it starts (``nginx-common`` and ``nginx-core``, ``postgresql-common`` and +``postgresql-16``) are what the unit is made of. Reading that is the same on +every systemd host -- ``systemctl show`` and ``dpkg -S`` or ``rpm -qf`` -- so the +command and its parse live here once, and a driver only carries the command. + +Only the unit's own packages: the libraries its binary links against are every +service's at once, and a decision about nginx is not one about libc. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess + +import pytest + +from napalm_device_types import ServicePackagesMixin +from napalm_device_types.service_packages import ( + SERVICE_PACKAGES_COMMAND, + parse_service_packages, +) + +#: ``systemctl show`` prints the properties in its own order: the name may follow its paths. +NGINX = """FragmentPath=/lib/systemd/system/nginx.service +ExecStart={ path=/usr/sbin/nginx ; argv[]=/usr/sbin/nginx -g daemon on; master_process on; ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Id=nginx.service +""" +CRON = """Id=cron.service +FragmentPath=/usr/lib/systemd/system/cron.service +ExecStart={ path=/usr/sbin/cron ; argv[]=/usr/sbin/cron -f $EXTRA_OPTS ; ignore_errors=no } +""" + + +def _wire(units: str, owners: str = "", *, tool: str = "dpkg", noise: str = "") -> str: + return f"{noise}SVCPKG_BEGIN\n[units]\n{units}\n[{tool}]\n{owners}SVCPKG_END\n" + + +DPKG = """nginx-common: /lib/systemd/system/nginx.service +nginx-core: /usr/sbin/nginx +cron: /lib/systemd/system/cron.service +cron: /usr/sbin/cron +""" + + +class TestParsing: + def test_the_unit_file_and_the_binary_name_its_packages(self): + assert parse_service_packages(_wire(NGINX, DPKG)) == { + "nginx": ["nginx-common", "nginx-core"] + } + + def test_dpkg_is_asked_across_the_usr_merge(self): + """dpkg knows ``/lib/systemd/system/cron.service``; the unit says ``/usr/lib/...``.""" + assert parse_service_packages(_wire(CRON, DPKG)) == {"cron": ["cron"]} + + def test_several_units_are_kept_apart(self): + found = parse_service_packages(_wire(NGINX + "\n" + CRON, DPKG)) + + assert found == {"nginx": ["nginx-common", "nginx-core"], "cron": ["cron"]} + + def test_rpm_names_its_packages_too(self): + units = """Id=postgresql-16.service +FragmentPath=/usr/lib/systemd/system/postgresql-16.service +ExecStart={ path=/usr/pgsql-16/bin/postmaster ; argv[]=/usr/pgsql-16/bin/postmaster -D ${PGDATA} } +""" + owners = ( + "/usr/lib/systemd/system/postgresql-16.service\tpostgresql16-server \n" + "/lib/systemd/system/postgresql-16.service\tfile /lib/systemd/system/postgresql-16.service is not owned by any package\n" + "/usr/pgsql-16/bin/postmaster\tpostgresql16-server \n" + ) + + found = parse_service_packages(_wire(units, owners, tool="rpm")) + + assert found == {"postgresql-16": ["postgresql16-server"]} + + def test_an_architecture_and_several_owners_are_read(self): + units = "Id=foo.service\nExecStart={ path=/usr/bin/foo ; argv[]=/usr/bin/foo }\n" + owners = "libfoo1:amd64, foo-bin: /usr/bin/foo\n" + + assert parse_service_packages(_wire(units, owners)) == {"foo": ["foo-bin", "libfoo1"]} + + def test_a_unit_no_package_owns_is_read_as_owning_none(self): + """A unit file an admin wrote: read, and nothing owns it -- not unknown.""" + units = "Id=local-backup.service\nFragmentPath=/etc/systemd/system/local-backup.service\n" + + assert parse_service_packages(_wire(units)) == {"local-backup": []} + + def test_a_diversion_is_no_owner(self): + units = "Id=x.service\nExecStart={ path=/bin/sh ; argv[]=/bin/sh -c x }\n" + owners = "diversion by dash from: /bin/sh\ndash: /bin/sh\n" + + assert parse_service_packages(_wire(units, owners)) == {"x": ["dash"]} + + def test_whatever_surrounds_the_frame_is_ignored(self): + noise = "Last login: today\nprintf '%s%s\\n' SVCPKG_ BEGIN; ...\n" + + found = parse_service_packages(_wire(NGINX, DPKG, noise=noise)) + + assert found == {"nginx": ["nginx-common", "nginx-core"]} + + def test_a_report_cut_short_raises(self): + """A list cut short must never read as services that own nothing.""" + with pytest.raises(ValueError): + parse_service_packages("SVCPKG_BEGIN\n[units]\n" + NGINX) + + +class TestTheCommand: + def test_the_frame_is_not_in_the_command_itself(self): + assert "SVCPKG_BEGIN" not in SERVICE_PACKAGES_COMMAND + assert "SVCPKG_END" not in SERVICE_PACKAGES_COMMAND + + def test_it_writes_nothing(self): + for verb in ("sudo", " > ", ">>", " rm ", "install", "restart"): + assert verb not in SERVICE_PACKAGES_COMMAND + + @pytest.mark.skipif( + not os.path.isdir("/run/systemd/system") or shutil.which("dpkg") is None, + reason="reads a systemd host with dpkg", + ) + def test_it_runs_and_parses_on_this_host(self): + out = subprocess.run( + ["sh", "-c", SERVICE_PACKAGES_COMMAND], capture_output=True, text=True, timeout=60 + ).stdout + + found = parse_service_packages(out) + + assert found, "a systemd host has services" + assert any(found.values()), "and a package owns at least one of them" + assert all(isinstance(p, str) and p for pkgs in found.values() for p in pkgs) + + +class TestTheMixin: + def test_it_hands_its_command_to_the_driver_and_parses_the_answer(self): + class Driver(ServicePackagesMixin): + def __init__(self): + self.sent = [] + + def _run_service_packages_command(self, command: str) -> str: + self.sent.append(command) + return _wire(NGINX, DPKG) + + driver = Driver() + + assert driver.get_service_packages() == {"nginx": ["nginx-common", "nginx-core"]} + assert driver.sent == [SERVICE_PACKAGES_COMMAND] From a21e365c4e4f91541140c2261b08518e9d3ff34e Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 19 Oct 2026 18:50:41 +0200 Subject: [PATCH 15/50] fix: annotate the unit path set so mypy accepts service_packages (NetOrk/netork#1236) --- napalm_device_types/service_packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/napalm_device_types/service_packages.py b/napalm_device_types/service_packages.py index c0dc146..f65068a 100644 --- a/napalm_device_types/service_packages.py +++ b/napalm_device_types/service_packages.py @@ -82,7 +82,8 @@ def _report(output: str) -> Dict[str, List[str]]: def _units(lines: List[str]) -> Dict[str, Set[str]]: """``systemctl show`` blocks -> ``{unit: its unit file and binary}``.""" units: Dict[str, Set[str]] = {} - unit, paths = "", set() + unit = "" + paths: Set[str] = set() for line in [*lines, ""]: key, _, value = line.partition("=") if not line.strip(): From f77091c489d4ef676237affe1ea84b3b52037043 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Wed, 21 Oct 2026 23:10:44 +0200 Subject: [PATCH 16/50] feat: HostFirewallMixin, a Linux host's own netfilter rules (NetOrk/netork#1344) get_host_firewall() reads `nft -j list ruleset` as root and returns every chain with its family, table, hook, priority and policy, and every rule as matches and a verdict. Rules iptables-nft wrote show their extensions in nft's JSON only as an opaque `xt`, so the tables iptables manages are taken from iptables-save instead; a legacy iptables is kept beside nft's tables, since both apply. A condition the parse does not model is an `unknown` match with its text, never left out. Without root neither tool can be read, and HostFirewallUnavailable is raised rather than a host without rules. A driver supplies only _run_host_firewall_command(command, privileged=). Version 4.21.0. --- README.md | 11 + napalm_device_types/__init__.py | 9 + napalm_device_types/host_firewall.py | 764 +++++++++++++++++++++++++++ napalm_device_types/models.py | 56 ++ pyproject.toml | 2 +- tests/test_host_firewall.py | 501 ++++++++++++++++++ 6 files changed, 1342 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/host_firewall.py create mode 100644 tests/test_host_firewall.py diff --git a/README.md b/README.md index 0b2ba42..e7c6e8f 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,17 @@ without `ss` is read with `netstat -lntup` (OpenWrt's busybox, old net-tools); o cgroup names the procd service (`/services//`). A host with neither raises `ListeningSocketsUnavailable`. +`HostFirewallMixin` (`get_host_firewall`) reads a Linux host's own netfilter rules the same +way: `nft -j list ruleset`, with the tables iptables-nft manages taken from `iptables-save` +instead, because nft shows their extensions (`-m conntrack`, `-m set`) only as an opaque `xt`. +A legacy iptables is a ruleset of its own and is kept beside nft's. Every chain says its family, +table, hook, priority and policy; every rule is a list of matches (`iifname`, `saddr`, `dport`, +`ct_state`, ...) and a verdict. A condition it does not model is an `unknown` match with its +text, never left out. A driver supplies `_run_host_firewall_command(command, privileged=)`; both +tools need root, so a host where neither could be read raises `HostFirewallUnavailable` rather +than reading as one without rules. Whether the rules keep a service unreachable is the +consumer's call. + `VirtualIpsMixin` (`get_virtual_ips`) reads the addresses a host's HA configuration lets float between machines: vip-manager (5.x's `--config` YAML, 1.x's `-ip=${VIP_IP}` resolved through the unit's environment) and keepalived (`virtual_ipaddress` of every diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index e9a3bf1..3c8028a 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -47,6 +47,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.interface_filter.InterfaceFilterMixin` * :class:`~napalm_device_types.kernel.KernelFactsMixin` * :class:`~napalm_device_types.listening.ListeningSocketsMixin` +* :class:`~napalm_device_types.host_firewall.HostFirewallMixin` * :class:`~napalm_device_types.mac_acl.MacAclMixin` * :class:`~napalm_device_types.nat_vpn.NatVpnMixin` * :class:`~napalm_device_types.packages.PackageManagementMixin` @@ -105,6 +106,11 @@ from napalm_device_types.service_packages import ( parse_service_packages, ) from napalm_device_types.lag import add_lag_interfaces +from napalm_device_types.host_firewall import ( + HostFirewallMixin, + HostFirewallUnavailable, + parse_host_firewall, +) from napalm_device_types.listening import ( LISTENING_SOCKETS_COMMAND, ListeningSocketsMixin, @@ -275,6 +281,9 @@ __all__ = [ "ServicePackagesMixin", "parse_service_packages", "LISTENING_SOCKETS_COMMAND", + "HostFirewallMixin", + "HostFirewallUnavailable", + "parse_host_firewall", "ListeningSocketsMixin", "ListeningSocketsUnavailable", "parse_listening_sockets", diff --git a/napalm_device_types/host_firewall.py b/napalm_device_types/host_firewall.py new file mode 100644 index 0000000..06211e4 --- /dev/null +++ b/napalm_device_types/host_firewall.py @@ -0,0 +1,764 @@ +# -*- coding: utf-8 -*- +"""A Linux host's own firewall: its netfilter rules, read once, in one shape. + +Whether a service is reachable from outside its host is decided twice: by the +address it listens on (:mod:`napalm_device_types.listening`), and by the host's +own firewall, which may drop the packets first. Reading the firewall is the +same on every Linux host, so the command and its parse live here, and a driver +only carries the command across. + +**What is read.** ``nft -j list ruleset`` is the primary reading: every table, +the host's own ``nftables.service`` ones included. Rules that iptables-nft wrote +are in it too, but their extensions show only as an opaque ``xt`` -- a +``-m conntrack --ctstate`` or ``-m set`` says nothing there. ``iptables-save`` +spells them out, so the tables iptables manages are taken from it, and the +others from nft. A *legacy* iptables is a second ruleset beside nft, and both +apply to every packet, so then both are kept whole. + +**One shape.** Each chain says its family, table, hook, priority and policy; +each rule a list of matches and a verdict. A match is ``field``, ``op`` +(``==``/``!=``) and ``values``. The fields are the few that decide reachability +from outside: ``iifname``, ``oifname``, ``saddr``, ``daddr``, ``l4proto``, +``dport``, ``sport``, ``ct_state``, ``ct_orig_dport`` and ``nfproto``. +Everything else -- an ipset, ``addrtype``, a rate limit, a ``fib`` lookup -- is +``unknown`` with its text, never dropped: a rule that matches on something this +does not model must not read as one that matches everything. + +**Root, and only root.** Both tools need ``CAP_NET_ADMIN``; without it they +print an error, not an empty ruleset. So there is no reading without root: +:class:`HostFirewallUnavailable` is raised rather than a host without rules. + +Whether a rule set makes a service unreachable is the consumer's call. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +import shlex +from shlex import quote +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING + +from napalm_device_types.models import ( + HostFirewallDict, + NetfilterChainDict, + NetfilterMatchDict, + NetfilterRuleDict, +) +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "FW_BEGIN" +_END = "FW_END" +_RC_RE = re.compile(r"^__FW_RC=(\d+)$") + +#: One line, POSIX ``sh``, read-only. The frame markers are printed in two +#: halves so that a transport which echoes the command does not show them early. +#: Both tools live in ``/usr/sbin`` on most systems, outside a login user's ``PATH``. +HOST_FIREWALL_COMMAND = ( + "PATH=$PATH:/usr/sbin:/sbin; " + "printf '%s%s\\n' FW_ BEGIN; " + "if command -v nft >/dev/null 2>&1; then " + "echo '[nft]'; nft -j list ruleset 2>&1; echo \"__FW_RC=$?\"; fi; " + "for t in iptables-save ip6tables-save; do " + "if command -v $t >/dev/null 2>&1; then " + 'echo "[$t]"; $t 2>&1; echo "__FW_RC=$?"; fi; done; ' + "printf '%s%s\\n' FW_ END" +) + +#: ``iptables-save`` reads the IPv4 tables, ``ip6tables-save`` the IPv6 ones. +_IPTABLES_FAMILY = {"iptables-save": "ip", "ip6tables-save": "ip6"} + +#: The built-in chains of iptables' tables: ``(type, hook, priority)``, as +#: iptables-nft registers them (and as the legacy tables run in the kernel). +_BUILT_IN: Dict[str, Dict[str, Tuple[str, str, int]]] = { + "raw": {"PREROUTING": ("filter", "prerouting", -300), "OUTPUT": ("filter", "output", -300)}, + "mangle": { + "PREROUTING": ("filter", "prerouting", -150), + "INPUT": ("filter", "input", -150), + "FORWARD": ("filter", "forward", -150), + "OUTPUT": ("route", "output", -150), + "POSTROUTING": ("filter", "postrouting", -150), + }, + "nat": { + "PREROUTING": ("nat", "prerouting", -100), + "INPUT": ("nat", "input", 100), + "OUTPUT": ("nat", "output", -100), + "POSTROUTING": ("nat", "postrouting", 100), + }, + "filter": { + "INPUT": ("filter", "input", 0), + "FORWARD": ("filter", "forward", 0), + "OUTPUT": ("filter", "output", 0), + }, + "security": { + "INPUT": ("filter", "input", 50), + "FORWARD": ("filter", "forward", 50), + "OUTPUT": ("filter", "output", 50), + }, +} + +_TERMINAL = {"ACCEPT": "accept", "DROP": "drop", "REJECT": "reject", "RETURN": "return"} +#: Targets that look at a packet, mark it or log it, and let it go on. +_NON_TERMINAL = frozenset( + { + "LOG", + "NFLOG", + "ULOG", + "TRACE", + "MARK", + "CONNMARK", + "CT", + "NOTRACK", + "TCPMSS", + "CHECKSUM", + "AUDIT", + "SET", + "DSCP", + "TOS", + "TTL", + "HL", + "CLASSIFY", + "CONNSECMARK", + "SECMARK", + "IDLETIMER", + "LED", + "RATEEST", + } +) + + +class HostFirewallUnavailable(NotImplementedError): + """Nothing on the host could be read: no ``nft``, no ``iptables-save``, or no root.""" + + +def _frame(output: str) -> List[str]: + lines = [line.rstrip() for line in strip_terminal_codes(output).splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact host firewall report in the output") from None + return lines[start + 1 : end] + + +def _sections(lines: List[str]) -> Dict[str, Tuple[List[str], Optional[int]]]: + """Each tool's output and exit status.""" + sections: Dict[str, Tuple[List[str], Optional[int]]] = {} + name: Optional[str] = None + body: List[str] = [] + for line in lines: + text = line.strip() + if text in ("[nft]", "[iptables-save]", "[ip6tables-save]"): + name, body = text[1:-1], [] + sections[name] = (body, None) + continue + rc = _RC_RE.match(text) + if rc and name is not None: + sections[name] = (body, int(rc.group(1))) + name = None + continue + if name is not None: + body.append(line) + return sections + + +# ── iptables-save ──────────────────────────────────────────────────────────── + +#: Options that start a new part of a rule, outside any ``-m`` module. +_TOP_LEVEL = frozenset( + { + "-m", + "--match", + "-j", + "--jump", + "-g", + "--goto", + "-p", + "--protocol", + "-s", + "--source", + "-d", + "--destination", + "-i", + "--in-interface", + "-o", + "--out-interface", + "-f", + "--fragment", + "-c", + "--set-counters", + } +) +#: Modules whose options this reads; any other module is an ``unknown`` match. +_KNOWN_MODULES = frozenset( + {"tcp", "udp", "sctp", "udplite", "multiport", "state", "conntrack", "comment"} +) + + +def _match(field: str, negated: bool, values: List[str]) -> NetfilterMatchDict: + return {"field": field, "op": "!=" if negated else "==", "values": values} + + +def _unknown(text: str) -> NetfilterMatchDict: + return {"field": "unknown", "op": "==", "values": [text]} + + +def _iface(name: str) -> str: + """iptables writes a wildcard interface as ``eth+``; nft as ``eth*``.""" + return name[:-1] + "*" if name.endswith("+") else name + + +def _ports(spec: str) -> List[str]: + return [p.replace(":", "-") for p in spec.split(",") if p] + + +def _states(spec: str) -> List[str]: + return [s.lower() for s in spec.split(",") if s] + + +class _RuleTokens: + """The tokens of one ``-A`` line, read front to back.""" + + def __init__(self, tokens: List[str]): + self.tokens = tokens + self.i = 0 + + def more(self) -> bool: + return self.i < len(self.tokens) + + def peek(self, ahead: int = 0) -> Optional[str]: + j = self.i + ahead + return self.tokens[j] if j < len(self.tokens) else None + + def take(self) -> str: + token = self.tokens[self.i] + self.i += 1 + return token + + +def _module_options(module: str, rt: _RuleTokens, matches: List[NetfilterMatchDict]) -> None: + """Read *module*'s options from *rt* into *matches*; stop at the next top-level option.""" + text = ["-m", module] + unknown = module not in _KNOWN_MODULES + while rt.more(): + token = rt.peek() + if token in _TOP_LEVEL or (token == "!" and rt.peek(1) in _TOP_LEVEL): + break + negated = False + if token == "!": + rt.take() + negated = True + text.append("!") + option = rt.take() + value = rt.take() if rt.more() and not str(rt.peek()).startswith("-") else "" + text += [option] + ([value] if value else []) + if unknown: + continue + mapped = _module_match(module, option, value, negated) + if mapped is None: + unknown = True + elif mapped is not False: + matches.append(mapped) + if unknown and module != "comment": + matches.append(_unknown(" ".join(text))) + + +def _module_match(module: str, option: str, value: str, negated: bool) -> Any: + """The match one module option makes; False for one that matches nothing new; None if not read.""" + if module == "comment": + return False + if module in ("tcp", "udp", "sctp", "udplite"): + if option in ("--dport", "--destination-port"): + return _match("dport", negated, _ports(value)) + if option in ("--sport", "--source-port"): + return _match("sport", negated, _ports(value)) + return None + if module == "multiport": + if option in ("--dports", "--destination-ports"): + return _match("dport", negated, _ports(value)) + if option in ("--sports", "--source-ports"): + return _match("sport", negated, _ports(value)) + return None + if module == "state" and option == "--state": + return _match("ct_state", negated, _states(value)) + if module == "conntrack": + if option == "--ctstate": + return _match("ct_state", negated, _states(value)) + if option == "--ctorigdstport": + return _match("ct_orig_dport", negated, _ports(value)) + return None + + +def _target(name: str, chains: Set[str]) -> Tuple[str, Optional[str]]: + if name in _TERMINAL: + return _TERMINAL[name], None + if name in chains: + return "jump", name + if name in _NON_TERMINAL: + return "continue", name + return "other", name + + +def _iptables_rule(line: str, chains: Set[str]) -> Tuple[str, NetfilterRuleDict]: + """``-A CHAIN ...`` as ``(chain, rule)``.""" + tokens = shlex.split(line) + chain = tokens[1] + rt = _RuleTokens(tokens[2:]) + matches: List[NetfilterMatchDict] = [] + verdict, target = "continue", None + negated = False + simple = { + "-i": "iifname", + "--in-interface": "iifname", + "-o": "oifname", + "--out-interface": "oifname", + "-s": "saddr", + "--source": "saddr", + "-d": "daddr", + "--destination": "daddr", + "-p": "l4proto", + "--protocol": "l4proto", + } + while rt.more(): + token = rt.take() + if token == "!": + negated = True + continue + if token in simple: + field, value = simple[token], rt.take() + if field in ("iifname", "oifname"): + matches.append(_match(field, negated, [_iface(value)])) + elif field == "l4proto": + if value.lower() != "all": + matches.append(_match(field, negated, [value.lower()])) + else: + matches.append(_match(field, negated, value.split(","))) + elif token in ("-m", "--match"): + _module_options(rt.take(), rt, matches) + elif token in ("-j", "--jump"): + verdict, target = _target(rt.take(), chains) + break # the target's own options follow and match nothing + elif token in ("-g", "--goto"): + verdict, target = "goto", rt.take() + break + elif token in ("-c", "--set-counters"): + rt.take(), rt.take() + else: + matches.append(_unknown(token)) + negated = False + return chain, {"matches": matches, "verdict": verdict, "target": target, "text": line} + + +def _iptables_variant(lines: List[str]) -> Optional[str]: + """``iptables-nft`` or ``iptables-legacy`` from the header; None if nothing was saved.""" + for line in lines: + if line.startswith("# Generated by"): + return "iptables-nft" if "(nf_tables)" in line else "iptables-legacy" + return None + + +def _parse_iptables_save(lines: List[str], family: str) -> List[NetfilterChainDict]: + chains: List[NetfilterChainDict] = [] + table: Optional[str] = None + by_name: Dict[str, NetfilterChainDict] = {} + for raw in lines: + line = raw.strip() + if line.startswith("*"): + table, by_name = line[1:], {} + elif line.startswith(":") and table is not None: + name, policy = line[1:].split()[:2] + kind, hook, prio = _BUILT_IN.get(table, {}).get(name, (None, None, None)) + chain: NetfilterChainDict = { + "family": family, + "table": table, + "name": name, + "type": kind, + "hook": hook, + "priority": prio, + "policy": None if policy == "-" else policy.lower(), + "rules": [], + } + by_name[name] = chain + chains.append(chain) + elif line.startswith("-A ") and table is not None: + name, rule = _iptables_rule(line, set(by_name)) + if name in by_name: + by_name[name]["rules"].append(rule) + return chains + + +# ── nft -j ─────────────────────────────────────────────────────────────────── + +_VERDICTS = { + "accept": "accept", + "drop": "drop", + "reject": "reject", + "return": "return", + "continue": "continue", +} +#: Statements that act on a packet without deciding its fate. +_NO_OPS = frozenset( + { + "counter", + "log", + "mangle", + "notrack", + "set", + "flow", + "ct helper", + "synproxy", + "tcp option", + "dup", + "fwd", + } +) +#: Statements that pass only some packets on: what follows is not about every packet. +_PARTIAL = frozenset({"limit", "quota", "numgen", "hash", "ct count"}) +#: Verdicts that hand the packet elsewhere, or that this does not model. +_OTHER = frozenset({"queue", "dnat", "snat", "masquerade", "redirect", "tproxy", "vmap"}) +_TRANSPORTS = frozenset({"tcp", "udp", "sctp", "udplite", "dccp"}) + + +def _nft_scalar(value: Any) -> Optional[List[str]]: + """A right-hand side as values; None for one this does not model.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, str)): + return [str(value)] + if isinstance(value, dict): + if "prefix" in value: + return [f"{value['prefix']['addr']}/{value['prefix']['len']}"] + if "range" in value: + low, high = value["range"] + return [f"{low}-{high}"] + if "elem" in value: + return _nft_scalar(value["elem"].get("val")) + if "set" in value: + return _nft_values(value["set"]) + if isinstance(value, list): + return _nft_values(value) + return None + + +def _nft_values(items: Any) -> Optional[List[str]]: + items = items if isinstance(items, list) else [items] + out: List[str] = [] + for item in items: + values = _nft_scalar(item) + if values is None: + return None + out += values + return out + + +def _left_text(left: Any) -> str: + if isinstance(left, dict): + if "payload" in left: + return ( + f"{left['payload'].get('protocol', '')} {left['payload'].get('field', '')}".strip() + ) + if "meta" in left: + return f"meta {left['meta'].get('key', '')}" + if "ct" in left: + ct = left["ct"] + return f"ct {ct.get('dir', '')} {ct.get('key', '')}".replace(" ", " ") + key = next(iter(left), "") + return str(key) + return str(left) + + +def _right_text(right: Any) -> str: + values = _nft_scalar(right) + if values is None: + return json.dumps(right, sort_keys=True) + return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" + + +def _nft_field(left: Any) -> Tuple[Optional[str], Optional[str]]: + """``(field, implied l4proto)`` of a left-hand side; ``(None, None)`` if not modelled.""" + if not isinstance(left, dict): + return None, None + if "meta" in left: + key = left["meta"].get("key") + return { + "iifname": "iifname", + "iif": "iifname", + "oifname": "oifname", + "oif": "oifname", + "l4proto": "l4proto", + "nfproto": "nfproto", + }.get(key), None + if "payload" in left: + proto, field = left["payload"].get("protocol"), left["payload"].get("field") + if proto in ("ip", "ip6") and field in ("saddr", "daddr"): + return field, None + if (proto, field) in (("ip", "protocol"), ("ip6", "nexthdr")): + return "l4proto", None + if field in ("dport", "sport") and (proto in _TRANSPORTS or proto == "th"): + return field, proto if proto in _TRANSPORTS else None + return None, None + if "ct" in left: + key, direction = left["ct"].get("key"), left["ct"].get("dir") + if key == "state": + return "ct_state", None + if key == "proto-dst" and direction == "original": + return "ct_orig_dport", None + return None, None + + +def _nft_match( + match: Dict[str, Any], sets: Dict[Tuple[str, str, str], List[str]], where: Tuple[str, str] +) -> List[NetfilterMatchDict]: + left, op, right = match.get("left"), match.get("op"), match.get("right") + field, implied = _nft_field(left) + raw_text = ( + f"{_left_text(left)} {'' if op in ('==', 'in') else str(op) + ' '}{_right_text(right)}" + ) + if field is None or op not in ("==", "in", "!="): + return [_unknown(raw_text)] + if isinstance(right, str) and right.startswith("@"): + values = sets.get((where[0], where[1], right[1:])) + else: + values = _nft_scalar(right) + if values is None: + return [_unknown(raw_text)] + out: List[NetfilterMatchDict] = [] + if implied: + out.append(_match("l4proto", False, [implied])) + if field in ("iifname", "oifname", "l4proto", "ct_state", "nfproto"): + values = [ + str(v).lower() if field != "iifname" and field != "oifname" else str(v) for v in values + ] + out.append(_match(field, op == "!=", values)) + return out + + +def _nft_rule( + rule: Dict[str, Any], sets: Dict[Tuple[str, str, str], List[str]] +) -> NetfilterRuleDict: + where = (rule.get("family", ""), rule.get("table", "")) + matches: List[NetfilterMatchDict] = [] + verdict, target = "continue", None + for statement in rule.get("expr") or []: + if not isinstance(statement, dict) or not statement: + continue + key = next(iter(statement)) + body = statement[key] + if key == "match": + matches += _nft_match(body, sets, where) + elif key in _VERDICTS: + verdict = _VERDICTS[key] + break + elif key in ("jump", "goto"): + verdict, target = key, str((body or {}).get("target")) + break + elif key == "xt": + if (body or {}).get("type") == "target": + verdict, target = "other", str(body.get("name")) + break + matches.append(_unknown(f"xt match {(body or {}).get('name')}")) + elif key in _PARTIAL: + matches.append(_unknown(key)) + elif key in _OTHER: + verdict, target = "other", key + break + elif key not in _NO_OPS: + matches.append(_unknown(key)) + entry: NetfilterRuleDict = { + "matches": matches, + "verdict": verdict, + "target": target, + "text": "", + } + entry["text"] = rule_text(entry) + return entry + + +def _parse_nft(text: str, skip: Set[Tuple[str, str]]) -> List[NetfilterChainDict]: + start = text.find("{") + if start < 0: + raise ValueError("nft printed no JSON") + objects = json.loads(text[start:]).get("nftables") or [] + sets: Dict[Tuple[str, str, str], List[str]] = {} + for obj in objects: + if "set" in obj and "elem" in obj["set"]: + values = _nft_values(obj["set"]["elem"]) + if values is not None: + s = obj["set"] + sets[(s.get("family", ""), s.get("table", ""), s.get("name", ""))] = values + chains: Dict[Tuple[str, str, str], NetfilterChainDict] = {} + for obj in objects: + if "chain" not in obj: + continue + c = obj["chain"] + if (c.get("family"), c.get("table")) in skip: + continue + chains[(c["family"], c["table"], c["name"])] = { + "family": c["family"], + "table": c["table"], + "name": c["name"], + "type": c.get("type"), + "hook": c.get("hook"), + "priority": c.get("prio"), + "policy": c.get("policy") if c.get("hook") else None, + "rules": [], + } + for obj in objects: + if "rule" not in obj: + continue + r = obj["rule"] + chain = chains.get((r.get("family"), r.get("table"), r.get("chain"))) + if chain is not None: + chain["rules"].append(_nft_rule(r, sets)) + return list(chains.values()) + + +# ── text ───────────────────────────────────────────────────────────────────── + + +def _family_of(values: List[str]) -> str: + try: + return ( + "ip6" + if all(ipaddress.ip_network(v, strict=False).version == 6 for v in values) + else "ip" + ) + except ValueError: + return "ip" + + +def _values_text(values: List[str]) -> str: + return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" + + +def rule_text(rule: NetfilterRuleDict) -> str: + """A rule as one nft-style line, from its normalized matches and verdict.""" + protos = [ + v + for m in rule["matches"] + if m["field"] == "l4proto" and m["op"] == "==" + for v in m["values"] + ] + parts: List[str] = [] + for m in rule["matches"]: + neg = "!= " if m["op"] == "!=" else "" + field, values = m["field"], m["values"] + if field == "unknown": + parts.append(" ".join(values)) + continue + keyword = { + "saddr": f"{_family_of(values)} saddr", + "daddr": f"{_family_of(values)} daddr", + "l4proto": "meta l4proto", + "nfproto": "meta nfproto", + "ct_state": "ct state", + "ct_orig_dport": "ct original proto-dst", + "dport": f"{protos[0] if len(protos) == 1 else 'th'} dport", + "sport": f"{protos[0] if len(protos) == 1 else 'th'} sport", + }.get(field, field) + parts.append(f"{keyword} {neg}{_values_text(values)}") + verdict = rule["verdict"] + if verdict in ("jump", "goto"): + parts.append(f"{verdict} {rule['target']}") + elif verdict == "other": + parts.append(str(rule["target"] or "other")) + elif verdict != "continue": + parts.append(verdict) + return " ".join(parts) + + +# ── the reading ────────────────────────────────────────────────────────────── + + +def _iptables_sections( + sections: Dict[str, Tuple[List[str], Optional[int]]], +) -> Iterator[Tuple[str, str, List[str]]]: + """``(variant, family, lines)`` for each iptables tool that saved something.""" + for tool, family in _IPTABLES_FAMILY.items(): + lines, rc = sections.get(tool, ([], None)) + if rc != 0: + continue + variant = _iptables_variant(lines) + if variant is not None: + yield variant, family, lines + + +def parse_host_firewall(output: str) -> HostFirewallDict: + """The framed report of :data:`HOST_FIREWALL_COMMAND` as one ruleset. + + :raises ValueError: if the report's frame is not intact. + :raises HostFirewallUnavailable: if neither tool could be read. + """ + sections = _sections(_frame(output)) + sources: List[str] = [] + chains: List[NetfilterChainDict] = [] + covered: Set[Tuple[str, str]] = set() + for variant, family, lines in _iptables_sections(sections): + saved = _parse_iptables_save(lines, family) + chains += saved + if variant not in sources: + sources.append(variant) + if variant == "iptables-nft": + covered |= {(c["family"], c["table"]) for c in saved} + nft_lines, nft_rc = sections.get("nft", ([], None)) + if nft_rc == 0: + try: + nft_chains = _parse_nft("\n".join(nft_lines), covered) + except ValueError: + nft_chains = None + if nft_chains is not None: + sources.insert(0, "nftables") + chains = nft_chains + chains + if not sources: + raise HostFirewallUnavailable("neither nft nor iptables-save could be read (root needed)") + return {"sources": sources, "chains": chains} + + +class HostFirewallMixin: + """Adds :meth:`get_host_firewall` to a driver that can run a command on a Linux host. + + The template form (README, "Function classes"): the command and its parse + are the same everywhere, so they are concrete here, and a driver supplies + only :meth:`_run_host_firewall_command` -- how a command reaches its host, + and how it gains root there. Mixed in by the drivers that can, so + ``hasattr(driver, "get_host_firewall")`` stays truthful. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_host_firewall_command(self, command: str, *, privileged: bool) -> str: + """Run *command* on the host and return what it printed; as root + when *privileged*. The command is a single ``sh -c`` invocation.""" + ... + + def get_host_firewall(self) -> HostFirewallDict: + """ + Returns the host's netfilter rules: every chain with its hook, priority + and policy, and every rule as matches and a verdict. + + * sources (list) - ``nftables``, ``iptables-nft``, ``iptables-legacy``: what was read + * chains (list) - see :class:`~napalm_device_types.models.NetfilterChainDict` + + Example:: + + { + "sources": ["nftables", "iptables-nft"], + "chains": [ + {"family": "ip", "table": "filter", "name": "INPUT", "type": "filter", + "hook": "input", "priority": 0, "policy": "accept", + "rules": [ + {"matches": [{"field": "iifname", "op": "==", "values": ["ens6"]}], + "verdict": "drop", "target": None, + "text": "-A INPUT -i ens6 -j DROP"}, + ]}, + ], + } + + :raises HostFirewallUnavailable: if neither tool could be read -- no + ``nft``, no ``iptables-save``, or no root to run them. + :raises ValueError: if the report came back cut short. + """ + command = f"sh -c {quote(HOST_FIREWALL_COMMAND)}" + return parse_host_firewall(self._run_host_firewall_command(command, privileged=True)) diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index d72634f..9ec1641 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -400,6 +400,62 @@ class ListeningSocketsDict(TypedDict): sockets: List[ListeningSocketDict] +class NetfilterMatchDict(TypedDict): + """One condition of a netfilter rule. + + ``field`` is one of ``iifname``, ``oifname``, ``saddr``, ``daddr``, + ``l4proto``, ``dport``, ``sport``, ``ct_state``, ``ct_orig_dport``, + ``nfproto`` -- or ``unknown``, whose single value is the condition's text. + The rule matches when the field's value is (``==``) or is not (``!=``) one + of ``values``: interface names (``eth*`` a wildcard), CIDRs, protocol names, + ports (``8000-8010`` a range), conntrack states in lower case. + """ + + field: str + op: str # "==" or "!=" + values: List[str] + + +class NetfilterRuleDict(TypedDict): + """One rule: every match must hold, then the verdict applies. + + ``verdict`` is ``accept``, ``drop``, ``reject``, ``return``, ``jump`` or + ``goto`` (to ``target``), ``continue`` (a rule that only counts, logs or + marks), or ``other`` -- NAT, a queue, a verdict map -- with ``target`` naming it. + """ + + matches: List[NetfilterMatchDict] + verdict: str + target: Optional[str] + text: str # the rule as iptables-save printed it, or as one nft-style line + + +class NetfilterChainDict(TypedDict): + """A chain. A base chain has a ``hook``, ``priority`` and ``policy``; a chain + reached only by ``jump``/``goto`` has None for all three.""" + + family: str # "ip", "ip6", "inet", "bridge", "arp", "netdev" + table: str + name: str + type: Optional[str] # "filter", "nat", "route" + hook: Optional[str] # "prerouting", "input", "forward", "output", "postrouting", ... + priority: Optional[int] + policy: Optional[str] # "accept" or "drop" + rules: List[NetfilterRuleDict] + + +class HostFirewallDict(TypedDict): + """What ``HostFirewallMixin.get_host_firewall`` read. + + ``sources`` names what was read: ``nftables``, ``iptables-nft`` (the tables + iptables manages, taken from ``iptables-save`` because nft shows their + extensions opaquely) and ``iptables-legacy`` (a ruleset of its own). + """ + + sources: List[str] + chains: List[NetfilterChainDict] + + class VirtualIpDict(TypedDict): """An address the host's HA configuration lets float between machines. diff --git a/pyproject.toml b/pyproject.toml index 7bb03aa..8b0471a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.20.0" +version = "4.21.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_host_firewall.py b/tests/test_host_firewall.py new file mode 100644 index 0000000..2bcd805 --- /dev/null +++ b/tests/test_host_firewall.py @@ -0,0 +1,501 @@ +"""get_host_firewall: a Linux host's own netfilter rules, read once, in one shape. + +Whether a service on a host is reachable from the internet is not only decided +by the address it listens on: the host's own firewall may drop the packets +before they reach it. Reading that is the same on every Linux host, so the +command and the parse live here, and a driver only carries the command across +(NetOrk/netork#1344). + +``nft -j list ruleset`` is the primary reading. Rules that iptables-nft wrote +show their extensions there only as an opaque ``xt`` (``-m conntrack``, +``-m set``), so the tables iptables manages are taken from ``iptables-save``, +which spells them out. A legacy iptables is a ruleset of its own beside nft, +and both apply. +""" + +from __future__ import annotations + +import json +from shlex import quote + +import pytest + +from napalm_device_types.host_firewall import ( + HOST_FIREWALL_COMMAND, + HostFirewallMixin, + HostFirewallUnavailable, + parse_host_firewall, +) + +IPTABLES_NFT = """# Generated by iptables-save v1.8.10 (nf_tables) on Wed Oct 21 14:52:11 2026 +*filter +:INPUT ACCEPT [2783578:295175610] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +:DOCKER-USER - [0:0] +:WGRUNNER - [0:0] +-A INPUT -s 10.255.255.4/32 -j WGRUNNER +-A INPUT -i ens6 -p udp -m udp --dport 51820 -j ACCEPT +-A INPUT -m set --match-set crowdsec-blacklists src -j DROP +-A INPUT -i ens6 -m state --state RELATED,ESTABLISHED -j ACCEPT +-A INPUT -i ens6 -p tcp -m multiport --dports 80,443,8000:8010 -j ACCEPT +-A INPUT ! -i lo -p tcp -m conntrack --ctstate NEW --ctorigdstport 2222 -j LOG --log-prefix "ssh " +-A INPUT -i ens6 -j DROP +-A FORWARD -j DOCKER-USER +-A DOCKER-USER -i eth+ ! -s 192.168.0.0/16 -g WGRUNNER +-A WGRUNNER -d 10.7.224.12/32 -p tcp -m tcp --dport 10051 -j ACCEPT +-A WGRUNNER -j REJECT --reject-with icmp-port-unreachable +-A WGRUNNER -j RETURN +COMMIT +# Completed on Wed Oct 21 14:52:11 2026 +*nat +:PREROUTING ACCEPT [0:0] +:POSTROUTING ACCEPT [0:0] +:DOCKER - [0:0] +-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER +-A DOCKER ! -i br-1 -p udp -m udp --dport 53 -j DNAT --to-destination 172.18.0.2:53 +COMMIT +""" + +IP6TABLES_NFT = """# Generated by ip6tables-save v1.8.10 (nf_tables) on Wed Oct 21 14:52:11 2026 +*filter +:INPUT DROP [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -i lo -j ACCEPT +COMMIT +""" + + +def _nft(*objects: dict) -> str: + meta = {"metainfo": {"version": "1.0.9", "json_schema_version": 1}} + return json.dumps({"nftables": [meta, *objects]}) + + +#: The iptables-nft filter table as nft shows it -- with its extensions opaque -- +#: and a native ``inet filter`` table of the host's own nftables.service. +NFT = _nft( + {"table": {"family": "ip", "name": "filter", "handle": 1}}, + { + "chain": { + "family": "ip", + "table": "filter", + "name": "INPUT", + "type": "filter", + "hook": "input", + "prio": 0, + "policy": "accept", + } + }, + { + "rule": { + "family": "ip", + "table": "filter", + "chain": "INPUT", + "expr": [ + {"xt": {"type": "match", "name": "set"}}, + {"counter": {"packets": 0, "bytes": 0}}, + {"drop": None}, + ], + } + }, + {"table": {"family": "inet", "name": "host", "handle": 9}}, + { + "set": { + "family": "inet", + "table": "host", + "name": "admins", + "type": "ipv4_addr", + "flags": ["interval"], + "elem": [{"prefix": {"addr": "198.51.100.0", "len": 24}}], + } + }, + { + "chain": { + "family": "inet", + "table": "host", + "name": "input", + "type": "filter", + "hook": "input", + "prio": 10, + "policy": "drop", + } + }, + {"chain": {"family": "inet", "table": "host", "name": "services"}}, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "in", + "left": {"ct": {"key": "state"}}, + "right": ["established", "related"], + } + }, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "==", "left": {"meta": {"key": "iif"}}, "right": "lo"}}, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "ip", "field": "saddr"}}, + "right": "@admins", + } + }, + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "tcp", "field": "dport"}}, + "right": {"set": [22, {"range": [8000, 8010]}]}, + } + }, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": "wg*"}}, + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "ip6", "field": "daddr"}}, + "right": {"prefix": {"addr": "2001:db8::", "len": 32}}, + } + }, + {"jump": {"target": "services"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "==", "left": {"meta": {"key": "l4proto"}}, "right": "udp"}}, + {"limit": {"rate": 10, "per": "second"}}, + {"log": {"prefix": "udp "}}, + {"reject": {"type": "icmpx", "expr": "port-unreachable"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "==", + "left": {"fib": {"result": "type", "flags": ["daddr"]}}, + "right": "local", + } + }, + {"goto": {"target": "services"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "services", + "expr": [ + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "udp", "field": "dport"}}, + "right": 53, + } + }, + {"queue": {"num": 1}}, + ], + } + }, +) + + +def _report( + *, + nft: str | None = NFT, + nft_rc: int = 0, + ipt: str | None = IPTABLES_NFT, + ip6t: str | None = IP6TABLES_NFT, +) -> str: + parts = ["$ sh -c ...", "FW_BEGIN"] + if nft is not None: + parts += ["[nft]", nft, f"__FW_RC={nft_rc}"] + if ipt is not None: + parts += ["[iptables-save]", ipt, "__FW_RC=0"] + if ip6t is not None: + parts += ["[ip6tables-save]", ip6t, "__FW_RC=0"] + parts.append("FW_END") + return "\n".join(parts) + "\n" + + +def _chain(reading, family, table, name): + (chain,) = [ + c + for c in reading["chains"] + if (c["family"], c["table"], c["name"]) == (family, table, name) + ] + return chain + + +def _fields(rule): + return [(m["field"], m["op"], m["values"]) for m in rule["matches"]] + + +class TestTheSources: + def test_iptables_tables_come_from_iptables_save_the_rest_from_nft(self): + reading = parse_host_firewall(_report()) + + assert reading["sources"] == ["nftables", "iptables-nft"] + tables = {(c["family"], c["table"]) for c in reading["chains"]} + assert tables == {("ip", "filter"), ("ip", "nat"), ("ip6", "filter"), ("inet", "host")} + # The filter table's INPUT is iptables-save's: its set match is spelled + # out there, and nft's opaque copy is not added a second time. + assert len(_chain(reading, "ip", "filter", "INPUT")["rules"]) == 7 + + def test_a_legacy_iptables_is_a_ruleset_of_its_own(self): + legacy = IPTABLES_NFT.replace("(nf_tables)", "(legacy)") + + reading = parse_host_firewall(_report(ipt=legacy, ip6t=None)) + + assert reading["sources"] == ["nftables", "iptables-legacy"] + inputs = [c for c in reading["chains"] if c["name"] == "INPUT" and c["family"] == "ip"] + assert len(inputs) == 2 # nft's own ip filter INPUT and legacy's both apply + + def test_nft_alone(self): + reading = parse_host_firewall(_report(ipt=None, ip6t=None)) + + assert reading["sources"] == ["nftables"] + assert _chain(reading, "ip", "filter", "INPUT")["rules"][0]["matches"] == [ + {"field": "unknown", "op": "==", "values": ["xt match set"]} + ] + + def test_iptables_alone_when_nft_fails(self): + reading = parse_host_firewall(_report(nft="Operation not permitted", nft_rc=1)) + + assert reading["sources"] == ["iptables-nft"] + + def test_no_tool_that_could_be_read(self): + with pytest.raises(HostFirewallUnavailable): + parse_host_firewall(_report(nft=None, ipt=None, ip6t=None)) + + def test_a_failed_reading_is_no_reading(self): + with pytest.raises(HostFirewallUnavailable): + parse_host_firewall(_report(nft="permission denied", nft_rc=1, ipt=None, ip6t=None)) + + def test_a_report_cut_short_raises(self): + with pytest.raises(ValueError): + parse_host_firewall(_report().replace("FW_END", "")) + + def test_an_empty_ruleset_is_a_host_without_rules(self): + reading = parse_host_firewall(_report(nft=_nft(), ipt=None, ip6t=None)) + + assert reading == {"sources": ["nftables"], "chains": []} + + +class TestIptablesSave: + def setup_method(self): + self.reading = parse_host_firewall(_report(nft=None)) + + def test_a_built_in_chain_has_its_hook_and_policy(self): + chain = _chain(self.reading, "ip", "filter", "INPUT") + assert (chain["type"], chain["hook"], chain["priority"], chain["policy"]) == ( + "filter", + "input", + 0, + "accept", + ) + assert _chain(self.reading, "ip6", "filter", "INPUT")["policy"] == "drop" + nat = _chain(self.reading, "ip", "nat", "PREROUTING") + assert (nat["type"], nat["hook"], nat["priority"]) == ("nat", "prerouting", -100) + + def test_a_user_chain_has_none(self): + chain = _chain(self.reading, "ip", "filter", "WGRUNNER") + assert (chain["type"], chain["hook"], chain["priority"], chain["policy"]) == ( + None, + None, + None, + None, + ) + + def test_matches_and_verdicts(self): + rules = _chain(self.reading, "ip", "filter", "INPUT")["rules"] + assert _fields(rules[0]) == [("saddr", "==", ["10.255.255.4/32"])] + assert (rules[0]["verdict"], rules[0]["target"]) == ("jump", "WGRUNNER") + assert _fields(rules[1]) == [ + ("iifname", "==", ["ens6"]), + ("l4proto", "==", ["udp"]), + ("dport", "==", ["51820"]), + ] + assert rules[1]["verdict"] == "accept" + assert _fields(rules[2]) == [ + ("unknown", "==", ["-m set --match-set crowdsec-blacklists src"]), + ] + assert _fields(rules[3]) == [ + ("iifname", "==", ["ens6"]), + ("ct_state", "==", ["related", "established"]), + ] + assert _fields(rules[4]) == [ + ("iifname", "==", ["ens6"]), + ("l4proto", "==", ["tcp"]), + ("dport", "==", ["80", "443", "8000-8010"]), + ] + assert _fields(rules[5]) == [ + ("iifname", "!=", ["lo"]), + ("l4proto", "==", ["tcp"]), + ("ct_state", "==", ["new"]), + ("ct_orig_dport", "==", ["2222"]), + ] + assert rules[5]["verdict"] == "continue" # LOG decides nothing + assert rules[6]["verdict"] == "drop" + + def test_negation_wildcard_and_goto(self): + (rule,) = _chain(self.reading, "ip", "filter", "DOCKER-USER")["rules"] + assert _fields(rule) == [("iifname", "==", ["eth*"]), ("saddr", "!=", ["192.168.0.0/16"])] + assert (rule["verdict"], rule["target"]) == ("goto", "WGRUNNER") + + def test_reject_return_and_nat_targets(self): + rules = _chain(self.reading, "ip", "filter", "WGRUNNER")["rules"] + assert [r["verdict"] for r in rules] == ["accept", "reject", "return"] + (dnat,) = _chain(self.reading, "ip", "nat", "DOCKER")["rules"] + assert (dnat["verdict"], dnat["target"]) == ("other", "DNAT") + assert _chain(self.reading, "ip", "nat", "PREROUTING")["rules"][0]["matches"] == [ + {"field": "unknown", "op": "==", "values": ["-m addrtype --dst-type LOCAL"]} + ] + + def test_the_line_is_kept_as_the_rules_text(self): + rule = _chain(self.reading, "ip", "filter", "INPUT")["rules"][6] + assert rule["text"] == "-A INPUT -i ens6 -j DROP" + + +class TestNftJson: + def setup_method(self): + self.chains = { + c["name"]: c + for c in parse_host_firewall(_report(ipt=None, ip6t=None))["chains"] + if c["table"] == "host" + } + + def test_a_base_chain_and_a_regular_one(self): + base, regular = self.chains["input"], self.chains["services"] + assert (base["family"], base["type"], base["hook"], base["priority"], base["policy"]) == ( + "inet", + "filter", + "input", + 10, + "drop", + ) + assert (regular["hook"], regular["policy"]) == (None, None) + + def test_matches(self): + rules = self.chains["input"]["rules"] + assert _fields(rules[0]) == [("ct_state", "==", ["established", "related"])] + assert _fields(rules[1]) == [("iifname", "==", ["lo"])] + # A named set is resolved from its elements; tcp dport implies tcp. + assert _fields(rules[2]) == [ + ("saddr", "==", ["198.51.100.0/24"]), + ("l4proto", "==", ["tcp"]), + ("dport", "==", ["22", "8000-8010"]), + ] + assert _fields(rules[3]) == [("iifname", "!=", ["wg*"]), ("daddr", "==", ["2001:db8::/32"])] + assert (rules[3]["verdict"], rules[3]["target"]) == ("jump", "services") + + def test_a_statement_that_takes_only_some_packets_is_unknown(self): + rule = self.chains["input"]["rules"][4] + assert _fields(rule) == [("l4proto", "==", ["udp"]), ("unknown", "==", ["limit"])] + assert rule["verdict"] == "reject" + + def test_an_expression_it_does_not_model_is_unknown(self): + rule = self.chains["input"]["rules"][5] + assert rule["matches"][0]["field"] == "unknown" + assert (rule["verdict"], rule["target"]) == ("goto", "services") + + def test_a_verdict_it_does_not_model_is_other(self): + (rule,) = self.chains["services"]["rules"] + assert rule["verdict"] == "other" + + def test_each_rule_reads_as_a_line(self): + assert self.chains["input"]["rules"][2]["text"] == ( + "ip saddr 198.51.100.0/24 meta l4proto tcp tcp dport {22, 8000-8010} accept" + ) + + def test_an_unresolvable_set_is_unknown(self): + nft = NFT.replace('"@admins"', '"@missing"') + rules = [ + c + for c in parse_host_firewall(_report(nft=nft, ipt=None, ip6t=None))["chains"] + if c["name"] == "input" + ][0]["rules"] + assert rules[2]["matches"][0] == { + "field": "unknown", + "op": "==", + "values": ["ip saddr @missing"], + } + + +class _Host(HostFirewallMixin): + def __init__(self, output: str | None = None, fail: Exception | None = None): + self.calls: list[tuple[str, bool]] = [] + self._output, self._fail = output, fail + + def _run_host_firewall_command(self, command: str, *, privileged: bool) -> str: + self.calls.append((command, privileged)) + if self._fail: + raise self._fail + return self._output or "" + + +class TestTheMixin: + def test_one_privileged_command(self): + host = _Host(_report()) + + reading = host.get_host_firewall() + + assert host.calls == [(f"sh -c {quote(HOST_FIREWALL_COMMAND)}", True)] + assert reading["sources"] == ["nftables", "iptables-nft"] + + def test_without_root_there_is_nothing_to_read(self): + # nft and iptables-save both need CAP_NET_ADMIN; a reading without root + # would be an empty ruleset that is not the host's. + host = _Host("FW_BEGIN\n[nft]\nnetlink: Operation not permitted\n__FW_RC=1\nFW_END\n") + + with pytest.raises(HostFirewallUnavailable): + host.get_host_firewall() + assert len(host.calls) == 1 + + def test_the_command_is_framed_and_read_only(self): + assert "nft -j list ruleset" in HOST_FIREWALL_COMMAND + assert "iptables-save" in HOST_FIREWALL_COMMAND + for word in (" -F", " -X", "flush", "delete", "-A ", "-I ", "restore"): + assert word not in HOST_FIREWALL_COMMAND From 08296d0d2eb8c3d3e4d5e2b8907807f080ae8389 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 22 Oct 2026 19:03:21 +0200 Subject: [PATCH 17/50] style: ruff format and the lint rules of the driver standard Mechanical: ruff format, and ruff check --fix with the rule set every NAPALM driver uses (E, F, I, UP, B at line length 100). typing.List/Dict/Optional become builtin generics and X | None, imports are sorted, the coding cookies are gone. Two tests that read the source text follow the new spelling. What ruff cannot fix comes in the next commit. --- README.md | 51 ++- napalm_device_types/__init__.py | 122 ++--- napalm_device_types/_matching.py | 14 +- napalm_device_types/_ucd_metrics.py | 143 +++--- napalm_device_types/access_point.py | 47 +- napalm_device_types/channel.py | 28 +- napalm_device_types/config_lifecycle.py | 13 +- napalm_device_types/config_sync.py | 1 - napalm_device_types/container_engine.py | 32 +- napalm_device_types/dhcp.py | 71 ++- napalm_device_types/dns_forward.py | 5 +- napalm_device_types/firewall.py | 38 +- napalm_device_types/firewall_rules.py | 24 +- napalm_device_types/firmware.py | 5 +- napalm_device_types/fixes.py | 19 +- napalm_device_types/health_metrics.py | 6 +- napalm_device_types/host_firewall.py | 84 ++-- napalm_device_types/host_status.py | 43 +- napalm_device_types/hypervisor.py | 44 +- napalm_device_types/interface_filter.py | 9 +- napalm_device_types/kernel.py | 15 +- napalm_device_types/lag.py | 15 +- napalm_device_types/licenses.py | 5 +- napalm_device_types/listening.py | 45 +- napalm_device_types/mac_acl.py | 7 +- napalm_device_types/management_controller.py | 7 +- napalm_device_types/media.py | 7 +- napalm_device_types/models.py | 440 ++++++++++--------- napalm_device_types/nat_vpn.py | 9 +- napalm_device_types/os.py | 27 +- napalm_device_types/ospf.py | 1 - napalm_device_types/package_updates.py | 14 +- napalm_device_types/packages.py | 9 +- napalm_device_types/phone.py | 7 +- napalm_device_types/ping_sweep.py | 25 +- napalm_device_types/residential_gateway.py | 19 +- napalm_device_types/roles.py | 9 +- napalm_device_types/service_clusters.py | 100 +++-- napalm_device_types/service_packages.py | 27 +- napalm_device_types/services.py | 7 +- napalm_device_types/ssh_host_keys.py | 26 +- napalm_device_types/ssh_server.py | 104 +++-- napalm_device_types/storage.py | 26 +- napalm_device_types/switch.py | 15 +- napalm_device_types/system_identity.py | 39 +- napalm_device_types/systemd.py | 39 +- napalm_device_types/terminal.py | 1 - napalm_device_types/updates.py | 9 +- napalm_device_types/virtual_ips.py | 53 +-- pyproject.toml | 7 + tests/test_channel.py | 4 +- tests/test_container_engine.py | 8 +- tests/test_dhcp_diff_apply.py | 15 +- tests/test_dhcp_ha_config_sync.py | 1 - tests/test_dhcp_subnet_diff_apply.py | 19 +- tests/test_fact_getters.py | 2 +- tests/test_fingerprint_base.py | 11 +- tests/test_firewall_diff_apply.py | 15 +- tests/test_fix_templates.py | 23 +- tests/test_host_status.py | 5 +- tests/test_lag_interfaces.py | 4 +- tests/test_listening_sockets.py | 4 +- tests/test_ping_sweep.py | 8 +- tests/test_port_forwards.py | 2 +- tests/test_provisioning.py | 1 + tests/test_role_contracts.py | 1 - tests/test_send_wake_on_lan.py | 1 + tests/test_service_clusters.py | 187 ++++++-- tests/test_ssh_host_keys.py | 4 +- tests/test_ssh_server.py | 107 +++-- tests/test_system_identity.py | 30 +- tests/test_systemd_services.py | 2 +- tests/test_type_label.py | 2 +- tests/test_ucd_metrics.py | 13 +- tests/test_virtual_ips.py | 16 +- tests/test_vm_cpu_types.py | 4 +- tests/test_vm_identifiers.py | 1 + 77 files changed, 1329 insertions(+), 1074 deletions(-) diff --git a/README.md b/README.md index e7c6e8f..1689662 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ Instead of inheriting directly from `napalm.base.NetworkDriver`, a driver can in ```python # without napalm-device-types -class OpenWrtDriver(NetworkDriver): - ... +class OpenWrtDriver(NetworkDriver): ... + # with napalm-device-types from napalm_device_types import AccessPointDriver -class OpenWrtDriver(AccessPointDriver): - ... + +class OpenWrtDriver(AccessPointDriver): ... ``` ## Why? @@ -29,8 +29,9 @@ OpenMediaVault box is a NAS built on Debian. So a driver inherits **one role bas role its device fills**, and **the order it lists them in is the ranking**: ```python -class QnapQtsDriver(StorageDriver, HypervisorDriver, LinuxDriver): - ... # primary_role_of(...) == "storage" +class QnapQtsDriver( + StorageDriver, HypervisorDriver, LinuxDriver +): ... # primary_role_of(...) == "storage" ``` `roles_of(cls)`, `role_keys_of(cls)` and `primary_role_of(cls)` read that back. Nothing @@ -54,10 +55,12 @@ host OS's load; it has no `HealthMetricsMixin` (SNMP) and no command channel. ```python class StorageDriver(DeviceTypeDriver): """Contract, not code.""" + ROLE: str = "storage" TYPE_LABEL: str = "Storage" - if TYPE_CHECKING: # nothing exists at runtime + if TYPE_CHECKING: # nothing exists at runtime + def get_disks(self) -> List[PhysicalDiskDict]: ... ``` @@ -312,20 +315,27 @@ generic data shapes belong here. Only the actual device communication belongs in **Worked example — firewall rule diff/apply** (`FirewallRuleMixin`, in `FirewallDriver`): ```python -class FirewallRuleMixin: # mixed into FirewallDriver +class FirewallRuleMixin: # mixed into FirewallDriver # Declared only — every driver implements its own device communication. if TYPE_CHECKING: + def get_firewall_rules(self) -> List[FirewallRuleDict]: ... - def apply_firewall_rule(self, rule: FirewallRuleDict, *, uuid: Optional[str] = None) -> Dict[str, Any]: ... + def apply_firewall_rule( + self, rule: FirewallRuleDict, *, uuid: Optional[str] = None + ) -> Dict[str, Any]: ... def commit_firewall_rules(self) -> Dict[str, Any]: ... # Concrete — the matching/comparison/orchestration algorithm is identical # for every firewall vendor, so it lives here once. - def diff_firewall_rules(self, desired: List[FirewallRuleDict]) -> FirewallRuleDiffDict: - ... # matches self.get_firewall_rules() against `desired` by description + def diff_firewall_rules( + self, desired: List[FirewallRuleDict] + ) -> ( + FirewallRuleDiffDict + ): ... # matches self.get_firewall_rules() against `desired` by description - def apply_firewall_ruleset(self, desired: List[FirewallRuleDict]): - ... # computes the diff, calls apply_firewall_rule() per change, commits + def apply_firewall_ruleset( + self, desired: List[FirewallRuleDict] + ): ... # computes the diff, calls apply_firewall_rule() per change, commits ``` The same split applies to `DhcpServerMixin`: `get_dhcp_reservations`/ @@ -421,8 +431,8 @@ HTTP_FINGERPRINT = [ ```python from napalm_device_types import AccessPointDriver -class OpenWrtDriver(AccessPointDriver): +class OpenWrtDriver(AccessPointDriver): def get_wireless_clients(self): # return List[WirelessClientDict] ... @@ -437,8 +447,8 @@ class OpenWrtDriver(AccessPointDriver): ```python from napalm_device_types import SwitchDriver -class CiscoIOSDriver(SwitchDriver): +class CiscoIOSDriver(SwitchDriver): def get_spanning_tree(self): # return Dict[str, SpanningTreeDict] ... @@ -453,8 +463,8 @@ class CiscoIOSDriver(SwitchDriver): ```python from napalm_device_types import FirewallDriver -class PfSenseDriver(FirewallDriver): +class PfSenseDriver(FirewallDriver): def get_nat_translations(self): # return List[NATTranslationDict] ... @@ -474,14 +484,13 @@ class PfSenseDriver(FirewallDriver): ```python from napalm_device_types import HypervisorDriver -class ProxmoxDriver(HypervisorDriver): +class ProxmoxDriver(HypervisorDriver): def get_vms(self): # return List[VMDict] ... - def create_vm_snapshot(self, name, snapshot, description="", include_memory=False): - ... + def create_vm_snapshot(self, name, snapshot, description="", include_memory=False): ... def get_vm_cpu_types(self): # optional — return List[VMCpuTypeDict]: the CPU models a new VM may get @@ -505,8 +514,8 @@ unpacked; both are the same for every hypervisor. ```python from napalm_device_types import OSDriver -class LinuxDriver(OSDriver): +class LinuxDriver(OSDriver): def get_packages(self): # return List[PackageDict] ... @@ -533,8 +542,8 @@ class LinuxDriver(OSDriver): ```python from napalm_device_types import StorageDriver -class TrueNASDriver(StorageDriver): +class TrueNASDriver(StorageDriver): def get_disks(self): # return List[PhysicalDiskDict] ... diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 3c8028a..5dbda76 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -64,8 +64,8 @@ Introspection -- :func:`~napalm_device_types.roles.roles_of`, :func:`~napalm_device_types.roles.primary_role_of`. """ -from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec, rules_max_weight from napalm_device_types.access_point import AccessPointDriver +from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec, rules_max_weight from napalm_device_types.channel import ( ByteStream, CommandChannelMixin, @@ -82,10 +82,10 @@ from napalm_device_types.container_engine import ( ContainerEngineUnavailable, ) from napalm_device_types.dhcp import DhcpServerMixin, normalize_cidr, normalize_mac +from napalm_device_types.dns_forward import DnsForwardMixin from napalm_device_types.firewall import FirewallDriver -from napalm_device_types.hypervisor import HypervisorDriver -from napalm_device_types.os import OSDriver from napalm_device_types.firewall_rules import FirewallRuleMixin +from napalm_device_types.firmware import FirmwareOfferMixin from napalm_device_types.fixes import ( EnsurePackageMixin, PackageProxyMixin, @@ -93,24 +93,18 @@ from napalm_device_types.fixes import ( run_fix_steps, ) from napalm_device_types.health_metrics import HealthMetricsMixin -from napalm_device_types.dns_forward import DnsForwardMixin -from napalm_device_types.firmware import FirmwareOfferMixin -from napalm_device_types.ospf import OspfMixin -from napalm_device_types.licenses import LicenseMixin -from napalm_device_types.host_reboot import HostRebootMixin -from napalm_device_types.interface_filter import InterfaceFilterMixin -from napalm_device_types.kernel import KERNEL_FACTS_COMMAND, KernelFactsMixin, parse_kernel_facts -from napalm_device_types.service_packages import ( - SERVICE_PACKAGES_COMMAND, - ServicePackagesMixin, - parse_service_packages, -) -from napalm_device_types.lag import add_lag_interfaces from napalm_device_types.host_firewall import ( HostFirewallMixin, HostFirewallUnavailable, parse_host_firewall, ) +from napalm_device_types.host_reboot import HostRebootMixin +from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status +from napalm_device_types.hypervisor import HypervisorDriver +from napalm_device_types.interface_filter import InterfaceFilterMixin +from napalm_device_types.kernel import KERNEL_FACTS_COMMAND, KernelFactsMixin, parse_kernel_facts +from napalm_device_types.lag import add_lag_interfaces +from napalm_device_types.licenses import LicenseMixin from napalm_device_types.listening import ( LISTENING_SOCKETS_COMMAND, ListeningSocketsMixin, @@ -125,67 +119,66 @@ from napalm_device_types.models import ( ConfigSyncDict, DhcpHaDict, DhcpHaPeerDict, + DnsForwardDict, EventLogEntryDict, FirmwareDict, - HardwareInventoryDict, - ManagementProtocolsDict, - PowerAction, - PowerStateDict, - HypervisorHostDict, - DnsForwardDict, FirmwareOfferDict, - OspfDict, - OspfInterfaceDict, - UpdateProgressDict, - LicenseDict, FixResultDict, FixStepDict, FixStepOutcomeDict, + HardwareInventoryDict, + HypervisorHostDict, + LicenseDict, + ManagementProtocolsDict, + OspfDict, + OspfInterfaceDict, + PowerAction, + PowerStateDict, SnmpAgentSettingsDict, SnmpHealthProfile, SnmpInterfacesSpec, SnmpMemorySpec, SnmpValueSpec, + UpdateProgressDict, ) from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.packages import PackageManagementMixin -from napalm_device_types.phone import PhoneDriver -from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping -from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of -from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status +from napalm_device_types.os import OSDriver +from napalm_device_types.ospf import OspfMixin from napalm_device_types.package_updates import ( APT_UPGRADABLE_COMMAND, DNF_SECURITY_COMMAND, parse_apt_upgradable, parse_dnf_security, ) -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.terminal import strip_terminal_codes -from napalm_device_types.systemd import ( - SYSTEMD_SERVICES_COMMAND, - SystemdServicesMixin, - SystemdUnavailable, - parse_systemd_services, -) -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.system_identity import ( - SYSTEM_IDENTITY_COMMAND, - SystemIdentityMixin, - is_placeholder_identity, - parse_system_identity, -) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.phone import PhoneDriver +from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping +from napalm_device_types.residential_gateway import ResidentialGatewayDriver +from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of from napalm_device_types.service_clusters import ( SERVICE_CLUSTERS_COMMAND, ServiceClustersMixin, parse_service_clusters, service_clusters_command, ) -from napalm_device_types.virtual_ips import ( - VIRTUAL_IPS_COMMAND, - VirtualIpsMixin, - parse_virtual_ips, +from napalm_device_types.service_packages import ( + SERVICE_PACKAGES_COMMAND, + ServicePackagesMixin, + parse_service_packages, +) +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.ssh_host_keys import ( + KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, +) +from napalm_device_types.ssh_host_keys import ( + OBSERVER_ARG as SSH_HOST_KEY_OBSERVER_ARG, +) +from napalm_device_types.ssh_host_keys import ( + HostKeyCheck, + HostKeyObservation, + SshHostKey, + SshHostKeyMismatch, ) -from napalm_device_types.residential_gateway import ResidentialGatewayDriver from napalm_device_types.ssh_server import ( SSH_ALGORITHM_CLASSES, OpenSshServerMixin, @@ -193,16 +186,27 @@ from napalm_device_types.ssh_server import ( SshHardeningResult, plan_ssh_hardening, ) -from napalm_device_types.ssh_host_keys import ( - KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, - OBSERVER_ARG as SSH_HOST_KEY_OBSERVER_ARG, - HostKeyCheck, - HostKeyObservation, - SshHostKey, - SshHostKeyMismatch, -) from napalm_device_types.storage import StorageDriver from napalm_device_types.switch import SwitchDriver +from napalm_device_types.system_identity import ( + SYSTEM_IDENTITY_COMMAND, + SystemIdentityMixin, + is_placeholder_identity, + parse_system_identity, +) +from napalm_device_types.systemd import ( + SYSTEMD_SERVICES_COMMAND, + SystemdServicesMixin, + SystemdUnavailable, + parse_systemd_services, +) +from napalm_device_types.terminal import strip_terminal_codes +from napalm_device_types.updates import UpdateMixin +from napalm_device_types.virtual_ips import ( + VIRTUAL_IPS_COMMAND, + VirtualIpsMixin, + parse_virtual_ips, +) __all__ = [ "HypervisorHostDict", diff --git a/napalm_device_types/_matching.py b/napalm_device_types/_matching.py index 8628053..839c16e 100644 --- a/napalm_device_types/_matching.py +++ b/napalm_device_types/_matching.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Keying live and desired entries for a diff, refusing what cannot be matched. A diff matches a desired entry to a live one by a key (a rule's description, a @@ -8,18 +7,19 @@ name a single entry, and guessing would overwrite one the caller never meant. from __future__ import annotations -from typing import Callable, Dict, Iterable, List, Set, Tuple, TypeVar +from collections.abc import Callable, Iterable +from typing import TypeVar T = TypeVar("T") -def index_live(entries: Iterable[T], key: Callable[[T], str]) -> Tuple[Dict[str, T], Set[str]]: +def index_live(entries: Iterable[T], key: Callable[[T], str]) -> tuple[dict[str, T], set[str]]: """Live entries by key, and the keys more than one of them holds. Entries without a key are left out: they are not the caller's to match. """ - by_key: Dict[str, T] = {} - shared: Set[str] = set() + by_key: dict[str, T] = {} + shared: set[str] = set() for entry in entries: k = key(entry) if not k: @@ -32,10 +32,10 @@ def index_live(entries: Iterable[T], key: Callable[[T], str]) -> Tuple[Dict[str, return by_key, shared -def check_desired(entries: List[T], key: Callable[[T], str], shared: Set[str], what: str) -> None: +def check_desired(entries: list[T], key: Callable[[T], str], shared: set[str], what: str) -> None: """``ValueError`` for a desired entry with no key, a key given twice, or a key that names more than one live entry.""" - seen: Set[str] = set() + seen: set[str] = set() for entry in entries: k = key(entry) if not k: diff --git a/napalm_device_types/_ucd_metrics.py b/napalm_device_types/_ucd_metrics.py index 7148f9d..9dc44f0 100644 --- a/napalm_device_types/_ucd_metrics.py +++ b/napalm_device_types/_ucd_metrics.py @@ -1,41 +1,43 @@ """Shared SNMP metric collection helpers for UCD-MIB and IF-MIB.""" + from __future__ import annotations import asyncio import re -from typing import Any, Awaitable, Callable, Dict, Optional +from collections.abc import Awaitable, Callable +from typing import Any # ── UCD-MIB OIDs ────────────────────────────────────────────────────────────── -OID_SYS_UPTIME = "1.3.6.1.2.1.25.1.1.0" # hrSystemUptime (centiseconds) -OID_CPU_IDLE = "1.3.6.1.4.1.2021.11.11.0" # UCD ssCpuIdle (%) -OID_MEM_TOTAL = "1.3.6.1.4.1.2021.4.5.0" # memTotalReal (kB) -OID_MEM_FREE = "1.3.6.1.4.1.2021.4.6.0" # memAvailReal (kB) -OID_MEM_BUFFER = "1.3.6.1.4.1.2021.4.14.0" # memBuffer (kB) -OID_MEM_CACHED = "1.3.6.1.4.1.2021.4.15.0" # memCached (kB) -OID_SWAP_TOTAL = "1.3.6.1.4.1.2021.4.3.0" # memTotalSwap (kB) -OID_SWAP_AVAIL = "1.3.6.1.4.1.2021.4.4.0" # memAvailSwap (kB) -OID_LOAD_1 = "1.3.6.1.4.1.2021.10.1.3.1" # laLoad 1-min -OID_LOAD_5 = "1.3.6.1.4.1.2021.10.1.3.2" # laLoad 5-min -OID_LOAD_15 = "1.3.6.1.4.1.2021.10.1.3.3" # laLoad 15-min +OID_SYS_UPTIME = "1.3.6.1.2.1.25.1.1.0" # hrSystemUptime (centiseconds) +OID_CPU_IDLE = "1.3.6.1.4.1.2021.11.11.0" # UCD ssCpuIdle (%) +OID_MEM_TOTAL = "1.3.6.1.4.1.2021.4.5.0" # memTotalReal (kB) +OID_MEM_FREE = "1.3.6.1.4.1.2021.4.6.0" # memAvailReal (kB) +OID_MEM_BUFFER = "1.3.6.1.4.1.2021.4.14.0" # memBuffer (kB) +OID_MEM_CACHED = "1.3.6.1.4.1.2021.4.15.0" # memCached (kB) +OID_SWAP_TOTAL = "1.3.6.1.4.1.2021.4.3.0" # memTotalSwap (kB) +OID_SWAP_AVAIL = "1.3.6.1.4.1.2021.4.4.0" # memAvailSwap (kB) +OID_LOAD_1 = "1.3.6.1.4.1.2021.10.1.3.1" # laLoad 1-min +OID_LOAD_5 = "1.3.6.1.4.1.2021.10.1.3.2" # laLoad 5-min +OID_LOAD_15 = "1.3.6.1.4.1.2021.10.1.3.3" # laLoad 15-min # ── IF-MIB OIDs ─────────────────────────────────────────────────────────────── -OID_IF_DESCR = "1.3.6.1.2.1.2.2.1.2" -OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5" -OID_IF_IN_OCT = "1.3.6.1.2.1.2.2.1.10" +OID_IF_DESCR = "1.3.6.1.2.1.2.2.1.2" +OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5" +OID_IF_IN_OCT = "1.3.6.1.2.1.2.2.1.10" OID_IF_OUT_OCT = "1.3.6.1.2.1.2.2.1.16" -OID_IF_IN_ERR = "1.3.6.1.2.1.2.2.1.14" +OID_IF_IN_ERR = "1.3.6.1.2.1.2.2.1.14" OID_IF_OUT_ERR = "1.3.6.1.2.1.2.2.1.20" # ifXTable: 64-bit octets and Mbps. The 32-bit counters wrap every ~34 s at # 1 Gbit/s and ifSpeed stops at 4294 Mbps; an agent without ifXTable (SNMPv1) # falls back to them. -OID_IF_HC_IN_OCT = "1.3.6.1.2.1.31.1.1.1.6" +OID_IF_HC_IN_OCT = "1.3.6.1.2.1.31.1.1.1.6" OID_IF_HC_OUT_OCT = "1.3.6.1.2.1.31.1.1.1.10" OID_IF_HIGH_SPEED = "1.3.6.1.2.1.31.1.1.1.15" -IF_SKIP_DEFAULT = re.compile(r'^(lo|sit\d|tun\d|docker|veth|br-|virbr)', re.IGNORECASE) +IF_SKIP_DEFAULT = re.compile(r"^(lo|sit\d|tun\d|docker|veth|br-|virbr)", re.IGNORECASE) -SnmpGetFn = Callable[[str], Awaitable[Optional[str]]] -SnmpWalkFn = Callable[[str], Awaitable[Dict[str, str]]] +SnmpGetFn = Callable[[str], Awaitable[str | None]] +SnmpWalkFn = Callable[[str], Awaitable[dict[str, str]]] def ticks_to_seconds(raw: str | None) -> int | None: @@ -48,35 +50,35 @@ def ticks_to_seconds(raw: str | None) -> int | None: def build_if_metrics( - metrics: Dict[str, Any], - descr: Dict[str, str], - speed: Dict[str, str], - in_oct: Dict[str, str], - out_oct: Dict[str, str], - in_err: Dict[str, str], - out_err: Dict[str, str], + metrics: dict[str, Any], + descr: dict[str, str], + speed: dict[str, str], + in_oct: dict[str, str], + out_oct: dict[str, str], + in_err: dict[str, str], + out_err: dict[str, str], *, if_skip: re.Pattern[str] | None = None, tx_err_is_drop: bool = False, - hc_in_oct: Optional[Dict[str, str]] = None, - hc_out_oct: Optional[Dict[str, str]] = None, - high_speed: Optional[Dict[str, str]] = None, + hc_in_oct: dict[str, str] | None = None, + hc_out_oct: dict[str, str] | None = None, + high_speed: dict[str, str] | None = None, ) -> None: if if_skip is None: if_skip = IF_SKIP_DEFAULT - interfaces: Dict[str, Any] = {} + interfaces: dict[str, Any] = {} for idx, name in descr.items(): if if_skip.match(name): continue - iface: Dict[str, Any] = {"name": name} + iface: dict[str, Any] = {"name": name} try: - iface["rx_bytes"] = int((hc_in_oct or {}).get(idx) or in_oct.get(idx) or 0) - iface["tx_bytes"] = int((hc_out_oct or {}).get(idx) or out_oct.get(idx) or 0) - iface["rx_errors"] = int(in_err.get(idx, 0) or 0) + iface["rx_bytes"] = int((hc_in_oct or {}).get(idx) or in_oct.get(idx) or 0) + iface["tx_bytes"] = int((hc_out_oct or {}).get(idx) or out_oct.get(idx) or 0) + iface["rx_errors"] = int(in_err.get(idx, 0) or 0) tx_val = int(out_err.get(idx, 0) or 0) if tx_err_is_drop: iface["tx_errors"] = 0 - iface["tx_drops"] = tx_val + iface["tx_drops"] = tx_val else: iface["tx_errors"] = tx_val high = (high_speed or {}).get(idx) @@ -98,15 +100,33 @@ async def collect_ucd_metrics( *, tx_err_is_drop: bool = False, if_skip: re.Pattern[str] | None = None, -) -> Dict[str, Any]: +) -> dict[str, Any]: """Collect UCD-MIB system metrics + IF-MIB interface counters in parallel.""" ( - uptime_raw, cpu_idle_raw, - mem_total_raw, mem_free_raw, mem_buf_raw, mem_cache_raw, - swap_total_raw, swap_avail_raw, - load1_raw, load5_raw, load15_raw, - ), ( - descr, speed, in_oct, out_oct, in_err, out_err, hc_in_oct, hc_out_oct, high_speed, + ( + uptime_raw, + cpu_idle_raw, + mem_total_raw, + mem_free_raw, + mem_buf_raw, + mem_cache_raw, + swap_total_raw, + swap_avail_raw, + load1_raw, + load5_raw, + load15_raw, + ), + ( + descr, + speed, + in_oct, + out_oct, + in_err, + out_err, + hc_in_oct, + hc_out_oct, + high_speed, + ), ) = await asyncio.gather( asyncio.gather( snmp_get(OID_SYS_UPTIME), @@ -134,7 +154,7 @@ async def collect_ucd_metrics( ), ) - metrics: Dict[str, Any] = {} + metrics: dict[str, Any] = {} secs = ticks_to_seconds(uptime_raw) if secs is not None: @@ -149,14 +169,14 @@ async def collect_ucd_metrics( if mem_total_raw and mem_free_raw: try: total_kb = int(mem_total_raw) - free_kb = int(mem_free_raw) - buf_kb = int(mem_buf_raw) if mem_buf_raw else 0 + free_kb = int(mem_free_raw) + buf_kb = int(mem_buf_raw) if mem_buf_raw else 0 cache_kb = int(mem_cache_raw) if mem_cache_raw else 0 - used_kb = max(0, total_kb - free_kb - buf_kb - cache_kb) + used_kb = max(0, total_kb - free_kb - buf_kb - cache_kb) metrics["memory_total_bytes"] = total_kb * 1024 - metrics["memory_used_bytes"] = used_kb * 1024 - metrics["memory_percent"] = round(used_kb / total_kb * 100, 1) if total_kb else 0.0 - metrics["memory_free_bytes"] = free_kb * 1024 + metrics["memory_used_bytes"] = used_kb * 1024 + metrics["memory_percent"] = round(used_kb / total_kb * 100, 1) if total_kb else 0.0 + metrics["memory_free_bytes"] = free_kb * 1024 except ValueError: pass @@ -164,10 +184,10 @@ async def collect_ucd_metrics( try: stotal = int(swap_total_raw) savail = int(swap_avail_raw) - sused = stotal - savail + sused = stotal - savail metrics["swap_total_bytes"] = stotal * 1024 - metrics["swap_used_bytes"] = sused * 1024 - metrics["swap_percent"] = round(sused / stotal * 100, 1) if stotal else 0.0 + metrics["swap_used_bytes"] = sused * 1024 + metrics["swap_percent"] = round(sused / stotal * 100, 1) if stotal else 0.0 except ValueError: pass @@ -178,7 +198,18 @@ async def collect_ucd_metrics( except ValueError: pass - build_if_metrics(metrics, descr, speed, in_oct, out_oct, in_err, out_err, - if_skip=if_skip, tx_err_is_drop=tx_err_is_drop, - hc_in_oct=hc_in_oct, hc_out_oct=hc_out_oct, high_speed=high_speed) + build_if_metrics( + metrics, + descr, + speed, + in_oct, + out_oct, + in_err, + out_err, + if_skip=if_skip, + tx_err_is_drop=tx_err_is_drop, + hc_in_oct=hc_in_oct, + hc_out_oct=hc_out_oct, + high_speed=high_speed, + ) return metrics diff --git a/napalm_device_types/access_point.py b/napalm_device_types/access_point.py index b5dbc9f..94a1146 100644 --- a/napalm_device_types/access_point.py +++ b/napalm_device_types/access_point.py @@ -10,14 +10,12 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.mac_acl import MacAclMixin -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.interface_filter import InterfaceFilterMixin +from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.models import ( Dot1XConfigDict, FastTransitionConfigDict, @@ -26,9 +24,20 @@ from napalm_device_types.models import ( SSIDBridgeDict, WirelessConfigDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.updates import UpdateMixin -class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, InterfaceFilterMixin, DeviceTypeDriver): +class AccessPointDriver( + MacAclMixin, + UpdateMixin, + ServiceControlMixin, + PackageManagementMixin, + HealthMetricsMixin, + InterfaceFilterMixin, + DeviceTypeDriver, +): """ Abstract intermediate driver for wireless access points. @@ -42,11 +51,6 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa ROLE: str = "access_point" TYPE_LABEL: str = "Access Point" - - - - - # AP-specific methods (get_wireless_clients, get_ssids, get_radio_status, # get_interfaces, get_vlans, …) are intentionally NOT defined here. # AccessPointDriver sits AFTER the mixin classes in the MRO of concrete @@ -86,7 +90,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_fast_transition_config(self) -> Dict[str, FastTransitionConfigDict]: + def get_fast_transition_config(self) -> dict[str, FastTransitionConfigDict]: """ Returns the 802.11r Fast BSS Transition (FT) configuration per SSID. @@ -118,7 +122,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_mesh_config(self) -> Dict[str, MeshConfigDict]: + def get_mesh_config(self) -> dict[str, MeshConfigDict]: """ Returns the 802.11s mesh configuration per mesh interface. @@ -148,7 +152,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_mesh_peers(self) -> List[MeshPeerDict]: + def get_mesh_peers(self) -> list[MeshPeerDict]: """ Returns a list of currently active 802.11s mesh peers. @@ -178,7 +182,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_ssid_bridge_config(self) -> Dict[str, SSIDBridgeDict]: + def get_ssid_bridge_config(self) -> dict[str, SSIDBridgeDict]: """ Returns the Layer-2 bridging configuration for each SSID, i.e. which bridge interface and VLAN each SSID is mapped to. @@ -219,9 +223,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - - - def get_dot1x_config(self) -> Dict[str, Dot1XConfigDict]: + def get_dot1x_config(self) -> dict[str, Dot1XConfigDict]: """ Returns the 802.1X / WPA-Enterprise (RADIUS) configuration per SSID. @@ -268,12 +270,3 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa } """ ... - - - - - - - - - diff --git a/napalm_device_types/channel.py b/napalm_device_types/channel.py index 6b534c9..775e458 100644 --- a/napalm_device_types/channel.py +++ b/napalm_device_types/channel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The command channel: a public way to run a command on a host, or open a stream. A driver already reaches its host, over SSH, a REST API or WinRM. What it did @@ -26,9 +25,8 @@ stays with the driver. from __future__ import annotations -import socket import time -from typing import Any, NamedTuple, Optional, Protocol, Tuple, TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, cast #: How much of a stream's stderr is kept. Enough to tell "permission denied" #: from "no such command"; a stream that writes megabytes there keeps its tail. @@ -56,7 +54,7 @@ class ByteStream(Protocol): ``close_write`` closes the writing side. """ - def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes: ... + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: ... def write(self, data: bytes) -> None: ... @@ -65,7 +63,7 @@ class ByteStream(Protocol): def close(self) -> None: ... @property - def exit_status(self) -> Optional[int]: ... + def exit_status(self) -> int | None: ... @property def stderr(self) -> str: ... @@ -82,7 +80,7 @@ class CommandChannelMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: """Run *command* with the host's shell; *stdin*, if given, is written first.""" ... @@ -108,11 +106,11 @@ class ParamikoExecStream: while self._channel.recv_stderr_ready(): self._stderr = (self._stderr + self._channel.recv_stderr(_READ))[-self._limit :] - def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes: + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: self._channel.settimeout(timeout) try: return cast(bytes, self._channel.recv(max_bytes)) - except socket.timeout: + except TimeoutError: raise TimeoutError(f"no data within {timeout}s") from None finally: self._drain_stderr() @@ -127,7 +125,7 @@ class ParamikoExecStream: self._channel.close() @property - def exit_status(self) -> Optional[int]: + def exit_status(self) -> int | None: if not self._channel.exit_status_ready(): return None return cast(int, self._channel.recv_exit_status()) @@ -138,7 +136,7 @@ class ParamikoExecStream: return self._stderr.decode("utf-8", "replace") -def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: +def _collect(channel: Any, command: str, timeout: float) -> tuple[bytes, bytes]: out, err = b"", b"" deadline = time.monotonic() + timeout while True: @@ -153,7 +151,7 @@ def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: out += chunk while chunk := channel.recv_stderr(_READ): err += chunk - except socket.timeout: + except TimeoutError: raise TimeoutError(f"{command!r} did not finish within {timeout}s") from None return out, err elif time.monotonic() > deadline: @@ -163,7 +161,7 @@ def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: def run_on_transport( - transport: Any, command: str, *, stdin: Optional[bytes] = None, timeout: float = 60 + transport: Any, command: str, *, stdin: bytes | None = None, timeout: float = 60 ) -> CommandResult: """Run *command* on an exec channel of the paramiko *transport*.""" channel = transport.open_session() @@ -175,14 +173,16 @@ def run_on_transport( channel.shutdown_write() out, err = _collect(channel, command, timeout) return CommandResult( - out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), channel.recv_exit_status() + out.decode("utf-8", "replace"), + err.decode("utf-8", "replace"), + channel.recv_exit_status(), ) finally: channel.close() def open_stream_on_transport( - transport: Any, command: str, *, stdin_prefix: Optional[bytes] = None + transport: Any, command: str, *, stdin_prefix: bytes | None = None ) -> ParamikoExecStream: """Start *command* on an exec channel and return its stream. diff --git a/napalm_device_types/config_lifecycle.py b/napalm_device_types/config_lifecycle.py index a27b5dd..c205bda 100644 --- a/napalm_device_types/config_lifecycle.py +++ b/napalm_device_types/config_lifecycle.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- from __future__ import annotations import difflib -from typing import ClassVar, TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from napalm.base.exceptions import ( CommandErrorException, @@ -41,15 +40,11 @@ class ConfigLifecycleMixin: if TYPE_CHECKING: - def _get_running_config(self) -> str: - ... + def _get_running_config(self) -> str: ... - def commit_config(self, message: str = "", revert_in: int | None = None) -> None: - ... + def commit_config(self, message: str = "", revert_in: int | None = None) -> None: ... - def load_merge_candidate( - self, filename: str | None = None, config: str | None = None - ) -> None: + def load_merge_candidate(self, filename: str | None = None, config: str | None = None) -> None: if filename is not None: try: with open(filename) as fh: diff --git a/napalm_device_types/config_sync.py b/napalm_device_types/config_sync.py index 8238ca5..7990cb8 100644 --- a/napalm_device_types/config_sync.py +++ b/napalm_device_types/config_sync.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Configuration sync between the two hosts of an HA pair. A host that pushes its configuration to a peer (OPNsense's XMLRPC sync, say) diff --git a/napalm_device_types/container_engine.py b/napalm_device_types/container_engine.py index 743ab1d..f0e35f7 100644 --- a/napalm_device_types/container_engine.py +++ b/napalm_device_types/container_engine.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Access to a host's container engine, and nothing more. This package abstracts the connection, the driver builds it, and netOrk does @@ -29,13 +28,14 @@ stays truthful. from __future__ import annotations import shlex -from typing import Any, Dict, List, Optional, Sequence, TYPE_CHECKING, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, cast from napalm_device_types.channel import ByteStream, CommandResult from napalm_device_types.models import ContainerEngineDict, FixResultDict #: Engines this package knows how to reach, and the API each speaks. -ENGINES: Dict[str, str] = {"docker": "docker-engine"} +ENGINES: dict[str, str] = {"docker": "docker-engine"} _PROBE_TIMEOUT = 15 _CLI_TIMEOUT = 120 @@ -68,7 +68,7 @@ class ContainerEngineConnection: self, args: Sequence[str], *, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, timeout: float = _CLI_TIMEOUT, privileged: bool = False, ) -> CommandResult: @@ -77,9 +77,12 @@ class ContainerEngineConnection: *privileged* runs it as root, the way the driver gains root for any command: for a call the engine refused to the login user. """ - return cast(CommandResult, self._driver.run_command( - self._command(args), privileged=privileged, timeout=timeout, stdin=stdin - )) + return cast( + CommandResult, + self._driver.run_command( + self._command(args), privileged=privileged, timeout=timeout, stdin=stdin + ), + ) def stream_cli( self, args: Sequence[str], *, merge_stderr: bool = False, privileged: bool = False @@ -90,9 +93,12 @@ class ContainerEngineConnection: progress there (``compose up``); *privileged* as for :meth:`run_cli`. """ command = self._command(args) - return cast(ByteStream, self._driver.open_stream( - f"{command} 2>&1" if merge_stderr else command, privileged=privileged - )) + return cast( + ByteStream, + self._driver.open_stream( + f"{command} 2>&1" if merge_stderr else command, privileged=privileged + ), + ) class ContainerEngineMixin: @@ -106,7 +112,7 @@ class ContainerEngineMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: ... def open_stream(self, command: str, *, privileged: bool = False) -> ByteStream: ... @@ -121,7 +127,7 @@ class ContainerEngineMixin: """The host command that is *engine*'s CLI. Override where it is not on PATH.""" return engine - def container_engines(self) -> List[ContainerEngineDict]: + def container_engines(self) -> list[ContainerEngineDict]: """ Returns the container engines the host offers: @@ -132,7 +138,7 @@ class ContainerEngineMixin: user may use it is decided by the first API call, which says "permission denied" in the stream's stderr. """ - found: List[ContainerEngineDict] = [] + found: list[ContainerEngineDict] = [] for engine, api in ENGINES.items(): probe = shlex.join(["command", "-v", self._container_engine_binary(engine)]) result = self.run_command(probe, timeout=_PROBE_TIMEOUT) diff --git a/napalm_device_types/dhcp.py b/napalm_device_types/dhcp.py index 281157e..f1f72ef 100644 --- a/napalm_device_types/dhcp.py +++ b/napalm_device_types/dhcp.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """DHCP desired-state management, shared by firewalls and gateways. Covers two independent desired-state sets: @@ -24,16 +23,17 @@ takes DHCP down for an entire VLAN. from __future__ import annotations import re -from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( DhcpHaDict, - DhcpReservationDiffDict, DhcpReservationDict, + DhcpReservationDiffDict, DhcpReservationUpdateDict, - DhcpSubnetDiffDict, DhcpSubnetDict, + DhcpSubnetDiffDict, DhcpSubnetUpdateDict, ) @@ -58,7 +58,7 @@ _DHCP_SUBNET_COMPARE_FIELDS = ( _HEX_ONLY = re.compile(r"[^0-9a-f]") -def normalize_mac(mac: Optional[str]) -> str: +def normalize_mac(mac: str | None) -> str: """Reduces a MAC address to lowercase colon-separated form. Devices report MACs in whatever form their config store happens to use -- @@ -80,7 +80,7 @@ def normalize_mac(mac: Optional[str]) -> str: return ":".join(hex_digits[i : i + 2] for i in range(0, 12, 2)) -def normalize_cidr(cidr: Optional[str]) -> str: +def normalize_cidr(cidr: str | None) -> str: """Reduces a subnet CIDR to a canonical string for matching. Only whitespace and case are normalised -- deliberately not the network @@ -91,9 +91,7 @@ def normalize_cidr(cidr: Optional[str]) -> str: return (cidr or "").strip().lower() -def _subnet_field_differs( - field: str, live: DhcpSubnetDict, desired: DhcpSubnetDict -) -> bool: +def _subnet_field_differs(field: str, live: DhcpSubnetDict, desired: DhcpSubnetDict) -> bool: """Compares one subnet field, with `option_data` handled specially. For `option_data` only the options `desired` actually names are compared; @@ -105,9 +103,7 @@ def _subnet_field_differs( desired_options = desired.get("option_data") or {} live_options = live.get("option_data") or {} - return any( - live_options.get(option) != value for option, value in desired_options.items() - ) + return any(live_options.get(option) != value for option, value in desired_options.items()) class DhcpServerMixin: @@ -125,7 +121,7 @@ class DhcpServerMixin: if TYPE_CHECKING: - def get_dhcp_reservations(self) -> List[DhcpReservationDict]: + def get_dhcp_reservations(self) -> list[DhcpReservationDict]: """ Returns all static DHCP reservations currently configured on the device, across all subnets. @@ -139,8 +135,8 @@ class DhcpServerMixin: ... def apply_dhcp_reservation( - self, reservation: DhcpReservationDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single static DHCP reservation on the device. @@ -157,7 +153,7 @@ class DhcpServerMixin: """ ... - def get_dhcp_subnets(self) -> List[DhcpSubnetDict]: + def get_dhcp_subnets(self) -> list[DhcpSubnetDict]: """ Returns every DHCPv4 subnet the device serves, with its pools and per-subnet options. @@ -168,8 +164,8 @@ class DhcpServerMixin: ... def apply_dhcp_subnet( - self, subnet: DhcpSubnetDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, subnet: DhcpSubnetDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single DHCPv4 subnet on the device. @@ -189,7 +185,7 @@ class DhcpServerMixin: """ ... - def commit_dhcp_subnets(self) -> Dict[str, Any]: + def commit_dhcp_subnets(self) -> dict[str, Any]: """ Applies pending subnet changes (e.g. Kea's ``service/reconfigure``). @@ -204,7 +200,7 @@ class DhcpServerMixin: """ ... - def commit_dhcp_reservations(self) -> Dict[str, Any]: + def commit_dhcp_reservations(self) -> dict[str, Any]: """ Applies pending reservation changes (e.g. Kea's ``service/reconfigure``, or a dnsmasq reload). @@ -238,9 +234,7 @@ class DhcpServerMixin: # Generic, vendor-neutral algorithms. # ------------------------------------------------------------------ - def diff_dhcp_reservations( - self, desired: List[DhcpReservationDict] - ) -> DhcpReservationDiffDict: + def diff_dhcp_reservations(self, desired: list[DhcpReservationDict]) -> DhcpReservationDiffDict: """ Compares `desired` against the device's current reservations and returns what would need to change to reach that state. @@ -263,14 +257,15 @@ class DhcpServerMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired reservation cannot name a single live one. """ + def key(reservation: DhcpReservationDict) -> str: return normalize_mac(reservation.get("mac")) live_by_mac, shared = index_live(self.get_dhcp_reservations(), key) check_desired(desired, key, shared, "MAC") - add: List[DhcpReservationDict] = [] - update: List[DhcpReservationUpdateDict] = [] + add: list[DhcpReservationDict] = [] + update: list[DhcpReservationUpdateDict] = [] for desired_reservation in desired: live = live_by_mac.get(key(desired_reservation)) @@ -294,9 +289,7 @@ class DhcpServerMixin: return {"add": add, "update": update} - def apply_dhcp_reservationset( - self, desired: List[DhcpReservationDict] - ) -> Iterator[str]: + def apply_dhcp_reservationset(self, desired: list[DhcpReservationDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. @@ -319,8 +312,7 @@ class DhcpServerMixin: self.apply_dhcp_reservation(entry["reservation"], uuid=entry["uuid"]) fields = ", ".join(entry["changed_fields"]) yield ( - f"[update] {entry['reservation']['mac']} -> " - f"{entry['reservation']['ip']} ({fields})" + f"[update] {entry['reservation']['mac']} -> {entry['reservation']['ip']} ({fields})" ) if not diff["add"] and not diff["update"]: @@ -328,12 +320,9 @@ class DhcpServerMixin: return self.commit_dhcp_reservations() - yield ( - f"[commit] applied {len(diff['add'])} add(s), " - f"{len(diff['update'])} update(s)" - ) + yield (f"[commit] applied {len(diff['add'])} add(s), {len(diff['update'])} update(s)") - def diff_dhcp_subnets(self, desired: List[DhcpSubnetDict]) -> DhcpSubnetDiffDict: + def diff_dhcp_subnets(self, desired: list[DhcpSubnetDict]) -> DhcpSubnetDiffDict: """ Compares `desired` against the device's current subnets and returns what would need to change to reach that state. @@ -362,14 +351,15 @@ class DhcpServerMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired subnet cannot name a single live one. """ + def key(subnet: DhcpSubnetDict) -> str: return normalize_cidr(subnet.get("subnet")) live_by_cidr, shared = index_live(self.get_dhcp_subnets(), key) check_desired(desired, key, shared, "subnet") - add: List[DhcpSubnetDict] = [] - update: List[DhcpSubnetUpdateDict] = [] + add: list[DhcpSubnetDict] = [] + update: list[DhcpSubnetUpdateDict] = [] for desired_subnet in desired: live = live_by_cidr.get(key(desired_subnet)) @@ -393,7 +383,7 @@ class DhcpServerMixin: return {"add": add, "update": update} - def apply_dhcp_subnetset(self, desired: List[DhcpSubnetDict]) -> Iterator[str]: + def apply_dhcp_subnetset(self, desired: list[DhcpSubnetDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. @@ -421,7 +411,4 @@ class DhcpServerMixin: return self.commit_dhcp_subnets() - yield ( - f"[commit] applied {len(diff['add'])} add(s), " - f"{len(diff['update'])} update(s)" - ) + yield (f"[commit] applied {len(diff['add'])} add(s), {len(diff['update'])} update(s)") diff --git a/napalm_device_types/dns_forward.py b/napalm_device_types/dns_forward.py index 38a277b..491c2c5 100644 --- a/napalm_device_types/dns_forward.py +++ b/napalm_device_types/dns_forward.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Domains a device's DNS resolver forwards to another server. An application that makes the device's resolver ask another server for a zone @@ -9,7 +8,7 @@ Declared only; a driver whose device resolves DNS implements it. from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import DnsForwardDict @@ -17,7 +16,7 @@ from napalm_device_types.models import DnsForwardDict class DnsForwardMixin: if TYPE_CHECKING: # pragma: no cover - declared for type checkers only - def get_dns_forwards(self) -> List[DnsForwardDict]: + def get_dns_forwards(self) -> list[DnsForwardDict]: """Every domain the resolver forwards, enabled or not.""" ... diff --git a/napalm_device_types/firewall.py b/napalm_device_types/firewall.py index f955898..adbb120 100644 --- a/napalm_device_types/firewall.py +++ b/napalm_device_types/firewall.py @@ -10,21 +10,28 @@ Usage:: ... """ -from typing import Any, ClassVar, Dict, Iterator, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.packages import PackageManagementMixin -from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.dhcp import DhcpServerMixin from napalm_device_types.firewall_rules import FirewallRuleMixin +from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( SecurityZoneDict, SessionDict, ) +from napalm_device_types.nat_vpn import NatVpnMixin +from napalm_device_types.packages import PackageManagementMixin - -class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, FirewallRuleMixin, DhcpServerMixin, DeviceTypeDriver): +class FirewallDriver( + NatVpnMixin, + PackageManagementMixin, + HealthMetricsMixin, + FirewallRuleMixin, + DhcpServerMixin, + DeviceTypeDriver, +): """ Abstract intermediate driver for firewall/security devices. @@ -42,11 +49,9 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi # OPNsense reports drops in the out-error counter. _SNMP_TX_ERR_IS_DROP: ClassVar[bool] = True - if TYPE_CHECKING: - - def get_security_zones(self) -> Dict[str, SecurityZoneDict]: + def get_security_zones(self) -> dict[str, SecurityZoneDict]: """ Returns the security zone configuration. @@ -73,7 +78,7 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - def get_sessions(self) -> List[SessionDict]: + def get_sessions(self) -> list[SessionDict]: """ Returns a list of active connection sessions (stateful flows). @@ -103,8 +108,7 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - - def send_wake_on_lan(self, mac_address: str, interface: str = "") -> Dict[str, Any]: + def send_wake_on_lan(self, mac_address: str, interface: str = "") -> dict[str, Any]: """ Sends a Wake-on-LAN "magic packet" to wake a host on the network. @@ -142,19 +146,9 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - - - - - # ------------------------------------------------------------------ # Firewall rule diff/apply. get_firewall_rules/apply_firewall_rule/ # commit_firewall_rules are abstract (device communication); everything # else here is a concrete, vendor-neutral algorithm -- see README.md # "Design principle: generic vs. device-specific logic". # ------------------------------------------------------------------ - - - - - diff --git a/napalm_device_types/firewall_rules.py b/napalm_device_types/firewall_rules.py index 8858e8f..2324029 100644 --- a/napalm_device_types/firewall_rules.py +++ b/napalm_device_types/firewall_rules.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Firewall rule reconciliation: generic here, device communication in the driver. The split follows the rule in this package's README: matching, diffing and the @@ -14,7 +13,8 @@ working implementation inherited from elsewhere. from __future__ import annotations -from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( @@ -43,7 +43,7 @@ class FirewallRuleMixin: if TYPE_CHECKING: - def get_firewall_rules(self) -> List[FirewallRuleDict]: + def get_firewall_rules(self) -> list[FirewallRuleDict]: """ Returns all firewall filter rules currently configured on the device. @@ -55,9 +55,10 @@ class FirewallRuleMixin: firewall rules. """ ... + def apply_firewall_rule( - self, rule: FirewallRuleDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, rule: FirewallRuleDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single firewall filter rule on the device. @@ -73,7 +74,8 @@ class FirewallRuleMixin: :returns: A dict with at least ``{"success": bool}``. """ ... - def commit_firewall_rules(self) -> Dict[str, Any]: + + def commit_firewall_rules(self) -> dict[str, Any]: """ Applies pending firewall filter rule changes (e.g. reloads pf/pfctl, or whatever the device's equivalent of "Apply Changes" is). @@ -88,7 +90,7 @@ class FirewallRuleMixin: """ ... - def diff_firewall_rules(self, desired: List[FirewallRuleDict]) -> FirewallRuleDiffDict: + def diff_firewall_rules(self, desired: list[FirewallRuleDict]) -> FirewallRuleDiffDict: """ Compares `desired` against the device's current rules and returns what would need to change to reach that state. @@ -113,14 +115,15 @@ class FirewallRuleMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired rule cannot name a single live one. """ + def key(rule: FirewallRuleDict) -> str: return (rule.get("description") or "").strip() live_by_description, shared = index_live(self.get_firewall_rules(), key) check_desired(desired, key, shared, "description") - add: List[FirewallRuleDict] = [] - update: List[FirewallRuleUpdateDict] = [] + add: list[FirewallRuleDict] = [] + update: list[FirewallRuleUpdateDict] = [] for desired_rule in desired: live_rule = live_by_description.get(key(desired_rule)) @@ -143,7 +146,8 @@ class FirewallRuleMixin: ) return {"add": add, "update": update} - def apply_firewall_ruleset(self, desired: List[FirewallRuleDict]) -> Iterator[str]: + + def apply_firewall_ruleset(self, desired: list[FirewallRuleDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. diff --git a/napalm_device_types/firmware.py b/napalm_device_types/firmware.py index aed95e9..08b0af1 100644 --- a/napalm_device_types/firmware.py +++ b/napalm_device_types/firmware.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The firmware a vendor offers for a device. A fact for the caller (docs/STRUCTURE.md): a driver may know the newer firmware @@ -10,7 +9,7 @@ the offer is newer. Declared only; a driver that can tell implements it. from __future__ import annotations -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import FirmwareOfferDict @@ -18,7 +17,7 @@ from napalm_device_types.models import FirmwareOfferDict class FirmwareOfferMixin: if TYPE_CHECKING: # pragma: no cover - declared for type checkers only - def get_firmware_offer(self) -> Optional[FirmwareOfferDict]: + def get_firmware_offer(self) -> FirmwareOfferDict | None: """The firmware running and the newest the vendor offers. None when the driver cannot tell for this device (an unknown model, diff --git a/napalm_device_types/fixes.py b/napalm_device_types/fixes.py index 9656960..a3335d6 100644 --- a/napalm_device_types/fixes.py +++ b/napalm_device_types/fixes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Fix routines: changes an application wants made on a device, as templates. A template fixes the order of the steps; the driver knows how each step is done on @@ -22,7 +21,8 @@ They replace ``run_device_action`` (docs/STRUCTURE.md): from __future__ import annotations -from typing import Any, Callable, List, Optional, Sequence, Tuple, TYPE_CHECKING +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ( FixResultDict, @@ -33,7 +33,7 @@ from napalm_device_types.models import ( ) #: One step of a fix: its name and the call that does it. -FixStep = Tuple[str, Callable[[], FixStepOutcomeDict]] +FixStep = tuple[str, Callable[[], FixStepOutcomeDict]] def run_fix_steps(steps: Sequence[FixStep]) -> FixResultDict: @@ -41,7 +41,7 @@ def run_fix_steps(steps: Sequence[FixStep]) -> FixResultDict: A step that raises has failed, with the exception's message as its output. """ - done: List[FixStepDict] = [] + done: list[FixStepDict] = [] for name, step in steps: try: outcome = step() @@ -79,7 +79,7 @@ class SnmpAgentMixin: if TYPE_CHECKING: # pragma: no cover - declared for type checkers only - def get_management_peer(self) -> Optional[str]: + def get_management_peer(self) -> str | None: """The address the device sees this management connection come from (for SSH, the client side of ``SSH_CONNECTION``), or None when it cannot tell. A fact for the caller choosing ``allowed_sources``; optional -- @@ -90,7 +90,9 @@ class SnmpAgentMixin: def _install_snmp_agent(self) -> FixStepOutcomeDict: ... - def _write_snmp_agent_config(self, settings: SnmpAgentSettingsDict) -> FixStepOutcomeDict: ... + def _write_snmp_agent_config( + self, settings: SnmpAgentSettingsDict + ) -> FixStepOutcomeDict: ... def _allow_snmp_sources(self, settings: SnmpAgentSettingsDict) -> FixStepOutcomeDict: """Let ``settings["allowed_sources"]`` reach the agent; ``ok`` with nothing @@ -126,7 +128,7 @@ class EnsurePackageMixin: if TYPE_CHECKING: # pragma: no cover - def get_packages(self) -> List[PackageDict]: ... + def get_packages(self) -> list[PackageDict]: ... def install_package(self, name: str, version: str = "") -> None: ... @@ -140,7 +142,7 @@ class PackageProxyMixin: """Have the package manager fetch through *url*; an empty *url* removes it.""" ... - def get_package_proxy(self) -> Optional[str]: + def get_package_proxy(self) -> str | None: """The proxy the package manager fetches through now; None for none. A fact: whether it is the right one is the caller's to judge.""" @@ -154,4 +156,3 @@ __all__ = [ "SnmpAgentMixin", "run_fix_steps", ] - diff --git a/napalm_device_types/health_metrics.py b/napalm_device_types/health_metrics.py index 50a54a5..5066475 100644 --- a/napalm_device_types/health_metrics.py +++ b/napalm_device_types/health_metrics.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """SNMP health collection, shared by every device type that speaks UCD-MIB. **Deprecated** (docs/STRUCTURE.md): drivers do not collect over SNMP. The @@ -21,7 +20,8 @@ from __future__ import annotations import re import warnings -from typing import Any, Awaitable, Callable, ClassVar, Dict, cast +from collections.abc import Awaitable, Callable +from typing import Any, ClassVar, cast from napalm_device_types._ucd_metrics import IF_SKIP_DEFAULT, collect_ucd_metrics from napalm_device_types.models import HealthMetricsDict @@ -43,7 +43,7 @@ class HealthMetricsMixin: async def get_health_metrics( cls, snmp_get: Callable[..., Awaitable[Any]], - snmp_walk: Callable[..., Awaitable[Dict[str, Any]]], + snmp_walk: Callable[..., Awaitable[dict[str, Any]]], ) -> HealthMetricsDict: """Collect SNMP health metrics (CPU, memory, uptime, load, interfaces). diff --git a/napalm_device_types/host_firewall.py b/napalm_device_types/host_firewall.py index 06211e4..259f94b 100644 --- a/napalm_device_types/host_firewall.py +++ b/napalm_device_types/host_firewall.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """A Linux host's own firewall: its netfilter rules, read once, in one shape. Whether a service is reachable from outside its host is decided twice: by the @@ -37,8 +36,9 @@ import ipaddress import json import re import shlex +from collections.abc import Iterator from shlex import quote -from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ( HostFirewallDict, @@ -71,7 +71,7 @@ _IPTABLES_FAMILY = {"iptables-save": "ip", "ip6tables-save": "ip6"} #: The built-in chains of iptables' tables: ``(type, hook, priority)``, as #: iptables-nft registers them (and as the legacy tables run in the kernel). -_BUILT_IN: Dict[str, Dict[str, Tuple[str, str, int]]] = { +_BUILT_IN: dict[str, dict[str, tuple[str, str, int]]] = { "raw": {"PREROUTING": ("filter", "prerouting", -300), "OUTPUT": ("filter", "output", -300)}, "mangle": { "PREROUTING": ("filter", "prerouting", -150), @@ -132,7 +132,7 @@ class HostFirewallUnavailable(NotImplementedError): """Nothing on the host could be read: no ``nft``, no ``iptables-save``, or no root.""" -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.rstrip() for line in strip_terminal_codes(output).splitlines()] stripped = [line.strip() for line in lines] try: @@ -143,11 +143,11 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> Dict[str, Tuple[List[str], Optional[int]]]: +def _sections(lines: list[str]) -> dict[str, tuple[list[str], int | None]]: """Each tool's output and exit status.""" - sections: Dict[str, Tuple[List[str], Optional[int]]] = {} - name: Optional[str] = None - body: List[str] = [] + sections: dict[str, tuple[list[str], int | None]] = {} + name: str | None = None + body: list[str] = [] for line in lines: text = line.strip() if text in ("[nft]", "[iptables-save]", "[ip6tables-save]"): @@ -197,7 +197,7 @@ _KNOWN_MODULES = frozenset( ) -def _match(field: str, negated: bool, values: List[str]) -> NetfilterMatchDict: +def _match(field: str, negated: bool, values: list[str]) -> NetfilterMatchDict: return {"field": field, "op": "!=" if negated else "==", "values": values} @@ -210,25 +210,25 @@ def _iface(name: str) -> str: return name[:-1] + "*" if name.endswith("+") else name -def _ports(spec: str) -> List[str]: +def _ports(spec: str) -> list[str]: return [p.replace(":", "-") for p in spec.split(",") if p] -def _states(spec: str) -> List[str]: +def _states(spec: str) -> list[str]: return [s.lower() for s in spec.split(",") if s] class _RuleTokens: """The tokens of one ``-A`` line, read front to back.""" - def __init__(self, tokens: List[str]): + def __init__(self, tokens: list[str]): self.tokens = tokens self.i = 0 def more(self) -> bool: return self.i < len(self.tokens) - def peek(self, ahead: int = 0) -> Optional[str]: + def peek(self, ahead: int = 0) -> str | None: j = self.i + ahead return self.tokens[j] if j < len(self.tokens) else None @@ -238,7 +238,7 @@ class _RuleTokens: return token -def _module_options(module: str, rt: _RuleTokens, matches: List[NetfilterMatchDict]) -> None: +def _module_options(module: str, rt: _RuleTokens, matches: list[NetfilterMatchDict]) -> None: """Read *module*'s options from *rt* into *matches*; stop at the next top-level option.""" text = ["-m", module] unknown = module not in _KNOWN_MODULES @@ -291,7 +291,7 @@ def _module_match(module: str, option: str, value: str, negated: bool) -> Any: return None -def _target(name: str, chains: Set[str]) -> Tuple[str, Optional[str]]: +def _target(name: str, chains: set[str]) -> tuple[str, str | None]: if name in _TERMINAL: return _TERMINAL[name], None if name in chains: @@ -301,12 +301,12 @@ def _target(name: str, chains: Set[str]) -> Tuple[str, Optional[str]]: return "other", name -def _iptables_rule(line: str, chains: Set[str]) -> Tuple[str, NetfilterRuleDict]: +def _iptables_rule(line: str, chains: set[str]) -> tuple[str, NetfilterRuleDict]: """``-A CHAIN ...`` as ``(chain, rule)``.""" tokens = shlex.split(line) chain = tokens[1] rt = _RuleTokens(tokens[2:]) - matches: List[NetfilterMatchDict] = [] + matches: list[NetfilterMatchDict] = [] verdict, target = "continue", None negated = False simple = { @@ -351,7 +351,7 @@ def _iptables_rule(line: str, chains: Set[str]) -> Tuple[str, NetfilterRuleDict] return chain, {"matches": matches, "verdict": verdict, "target": target, "text": line} -def _iptables_variant(lines: List[str]) -> Optional[str]: +def _iptables_variant(lines: list[str]) -> str | None: """``iptables-nft`` or ``iptables-legacy`` from the header; None if nothing was saved.""" for line in lines: if line.startswith("# Generated by"): @@ -359,10 +359,10 @@ def _iptables_variant(lines: List[str]) -> Optional[str]: return None -def _parse_iptables_save(lines: List[str], family: str) -> List[NetfilterChainDict]: - chains: List[NetfilterChainDict] = [] - table: Optional[str] = None - by_name: Dict[str, NetfilterChainDict] = {} +def _parse_iptables_save(lines: list[str], family: str) -> list[NetfilterChainDict]: + chains: list[NetfilterChainDict] = [] + table: str | None = None + by_name: dict[str, NetfilterChainDict] = {} for raw in lines: line = raw.strip() if line.startswith("*"): @@ -421,7 +421,7 @@ _OTHER = frozenset({"queue", "dnat", "snat", "masquerade", "redirect", "tproxy", _TRANSPORTS = frozenset({"tcp", "udp", "sctp", "udplite", "dccp"}) -def _nft_scalar(value: Any) -> Optional[List[str]]: +def _nft_scalar(value: Any) -> list[str] | None: """A right-hand side as values; None for one this does not model.""" if isinstance(value, bool): return None @@ -442,9 +442,9 @@ def _nft_scalar(value: Any) -> Optional[List[str]]: return None -def _nft_values(items: Any) -> Optional[List[str]]: +def _nft_values(items: Any) -> list[str] | None: items = items if isinstance(items, list) else [items] - out: List[str] = [] + out: list[str] = [] for item in items: values = _nft_scalar(item) if values is None: @@ -476,7 +476,7 @@ def _right_text(right: Any) -> str: return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" -def _nft_field(left: Any) -> Tuple[Optional[str], Optional[str]]: +def _nft_field(left: Any) -> tuple[str | None, str | None]: """``(field, implied l4proto)`` of a left-hand side; ``(None, None)`` if not modelled.""" if not isinstance(left, dict): return None, None @@ -509,8 +509,8 @@ def _nft_field(left: Any) -> Tuple[Optional[str], Optional[str]]: def _nft_match( - match: Dict[str, Any], sets: Dict[Tuple[str, str, str], List[str]], where: Tuple[str, str] -) -> List[NetfilterMatchDict]: + match: dict[str, Any], sets: dict[tuple[str, str, str], list[str]], where: tuple[str, str] +) -> list[NetfilterMatchDict]: left, op, right = match.get("left"), match.get("op"), match.get("right") field, implied = _nft_field(left) raw_text = ( @@ -524,7 +524,7 @@ def _nft_match( values = _nft_scalar(right) if values is None: return [_unknown(raw_text)] - out: List[NetfilterMatchDict] = [] + out: list[NetfilterMatchDict] = [] if implied: out.append(_match("l4proto", False, [implied])) if field in ("iifname", "oifname", "l4proto", "ct_state", "nfproto"): @@ -536,10 +536,10 @@ def _nft_match( def _nft_rule( - rule: Dict[str, Any], sets: Dict[Tuple[str, str, str], List[str]] + rule: dict[str, Any], sets: dict[tuple[str, str, str], list[str]] ) -> NetfilterRuleDict: where = (rule.get("family", ""), rule.get("table", "")) - matches: List[NetfilterMatchDict] = [] + matches: list[NetfilterMatchDict] = [] verdict, target = "continue", None for statement in rule.get("expr") or []: if not isinstance(statement, dict) or not statement: @@ -576,19 +576,19 @@ def _nft_rule( return entry -def _parse_nft(text: str, skip: Set[Tuple[str, str]]) -> List[NetfilterChainDict]: +def _parse_nft(text: str, skip: set[tuple[str, str]]) -> list[NetfilterChainDict]: start = text.find("{") if start < 0: raise ValueError("nft printed no JSON") objects = json.loads(text[start:]).get("nftables") or [] - sets: Dict[Tuple[str, str, str], List[str]] = {} + sets: dict[tuple[str, str, str], list[str]] = {} for obj in objects: if "set" in obj and "elem" in obj["set"]: values = _nft_values(obj["set"]["elem"]) if values is not None: s = obj["set"] sets[(s.get("family", ""), s.get("table", ""), s.get("name", ""))] = values - chains: Dict[Tuple[str, str, str], NetfilterChainDict] = {} + chains: dict[tuple[str, str, str], NetfilterChainDict] = {} for obj in objects: if "chain" not in obj: continue @@ -618,7 +618,7 @@ def _parse_nft(text: str, skip: Set[Tuple[str, str]]) -> List[NetfilterChainDict # ── text ───────────────────────────────────────────────────────────────────── -def _family_of(values: List[str]) -> str: +def _family_of(values: list[str]) -> str: try: return ( "ip6" @@ -629,7 +629,7 @@ def _family_of(values: List[str]) -> str: return "ip" -def _values_text(values: List[str]) -> str: +def _values_text(values: list[str]) -> str: return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" @@ -641,7 +641,7 @@ def rule_text(rule: NetfilterRuleDict) -> str: if m["field"] == "l4proto" and m["op"] == "==" for v in m["values"] ] - parts: List[str] = [] + parts: list[str] = [] for m in rule["matches"]: neg = "!= " if m["op"] == "!=" else "" field, values = m["field"], m["values"] @@ -673,8 +673,8 @@ def rule_text(rule: NetfilterRuleDict) -> str: def _iptables_sections( - sections: Dict[str, Tuple[List[str], Optional[int]]], -) -> Iterator[Tuple[str, str, List[str]]]: + sections: dict[str, tuple[list[str], int | None]], +) -> Iterator[tuple[str, str, list[str]]]: """``(variant, family, lines)`` for each iptables tool that saved something.""" for tool, family in _IPTABLES_FAMILY.items(): lines, rc = sections.get(tool, ([], None)) @@ -692,9 +692,9 @@ def parse_host_firewall(output: str) -> HostFirewallDict: :raises HostFirewallUnavailable: if neither tool could be read. """ sections = _sections(_frame(output)) - sources: List[str] = [] - chains: List[NetfilterChainDict] = [] - covered: Set[Tuple[str, str]] = set() + sources: list[str] = [] + chains: list[NetfilterChainDict] = [] + covered: set[tuple[str, str]] = set() for variant, family, lines in _iptables_sections(sections): saved = _parse_iptables_save(lines, family) chains += saved diff --git a/napalm_device_types/host_status.py b/napalm_device_types/host_status.py index ea257fa..68307fe 100644 --- a/napalm_device_types/host_status.py +++ b/napalm_device_types/host_status.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Host status: does the host need a reboot, and does it patch itself? A patch run that installed a new kernel or libc has not closed anything until @@ -59,7 +58,7 @@ too full is not decided here. from __future__ import annotations import re -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import ( FilesystemUsageDict, @@ -110,7 +109,7 @@ HOST_STATUS_COMMAND = ( "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; " # Named through a variable: spelt out, the program name would match BUSY_CHECK # in this very command line. - "uu=unattended-upgrade; { command -v \"$uu\" || [ -x \"/usr/bin/$uu\" ]; } >/dev/null 2>&1 " + 'uu=unattended-upgrade; { command -v "$uu" || [ -x "/usr/bin/$uu" ]; } >/dev/null 2>&1 ' "&& echo '[unattended-upgrade]'; " f"echo '[upgrade-stamp]'; stat -c %Y {_UPGRADE_STAMP} 2>/dev/null; fi; " "if command -v dpkg-query >/dev/null 2>&1; then " @@ -147,15 +146,15 @@ _HALFWAY = { } -def _sections(output: str) -> Dict[str, List[str]]: +def _sections(output: str) -> dict[str, list[str]]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) end = lines.index(_END, start) except ValueError: raise ValueError("no intact host status report in the output") from None - sections: Dict[str, List[str]] = {} - current: List[str] = [] + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines[start + 1 : end]: if line.startswith("[") and line.endswith("]"): current = sections.setdefault(line[1:-1], []) @@ -164,12 +163,12 @@ def _sections(output: str) -> Dict[str, List[str]]: return sections -def _version_key(version: str) -> Tuple[object, ...]: +def _version_key(version: str) -> tuple[object, ...]: """Natural order: 6.8.0-142 after 6.8.0-87, 7.0.14 after 7.0.2.""" return tuple(int(part) if part.isdigit() else part for part in re.split(r"(\d+)", version)) -def kernel_reboot_pending(running: str, installed: List[str]) -> Optional[str]: +def kernel_reboot_pending(running: str, installed: list[str]) -> str | None: """The newest installed kernel of the running flavour, if it is newer than the running one; otherwise None. @@ -187,7 +186,7 @@ def kernel_reboot_pending(running: str, installed: List[str]) -> Optional[str]: return None -def _kernel_flavour(release: str) -> Tuple[str, str]: +def _kernel_flavour(release: str) -> tuple[str, str]: """``6.8.0-87-generic`` -> ``("6.8.0-87", "generic")``, ``5.14.0-427.13.1.el9_4.x86_64`` -> ``("5.14.0-427.13.1.el9_4", "x86_64")``.""" base, sep, tail = release.rpartition("-") @@ -199,7 +198,7 @@ def _kernel_flavour(release: str) -> Tuple[str, str]: return base, tail -def _freebsd_reboot(sections: Dict[str, List[str]]) -> Optional[Tuple[bool, Optional[str]]]: +def _freebsd_reboot(sections: dict[str, list[str]]) -> tuple[bool, str | None] | None: """FreeBSD's answer from freebsd-version, or None on any other system.""" installed = (sections.get("freebsd-kernel") or [""])[0] running = (sections.get("freebsd-running") or [""])[0] @@ -210,7 +209,7 @@ def _freebsd_reboot(sections: Dict[str, List[str]]) -> Optional[Tuple[bool, Opti return False, None -def _reboot(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], Optional[str]]: +def _reboot(sections: dict[str, list[str]]) -> tuple[bool | None, str | None]: if "reboot-required" in sections: return True, f"{_REBOOT_FILE} is present" needs = sections.get("needs-restarting") @@ -229,29 +228,29 @@ def _reboot(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], Optional[st return None, None -def _timer_states(sections: Dict[str, List[str]]) -> Dict[str, str]: - states: Dict[str, str] = {} +def _timer_states(sections: dict[str, list[str]]) -> dict[str, str]: + states: dict[str, str] = {} for line in sections.get("timers") or []: unit, _, state = line.partition(" ") states[unit] = state.strip() return states -def _periodic(sections: Dict[str, List[str]]) -> Optional[str]: +def _periodic(sections: dict[str, list[str]]) -> str | None: """The value of ``APT::Periodic::Unattended-Upgrade``, or None when it is not set.""" lines = sections.get("apt-config") or [] match = next(filter(None, (_PERIODIC.match(line) for line in lines)), None) return match.group(1) if match else None -def _dnf_applies(sections: Dict[str, List[str]]) -> bool: +def _dnf_applies(sections: dict[str, list[str]]) -> bool: """Whether dnf-automatic.timer installs what it downloads; dnf's default is no.""" lines = sections.get("dnf-automatic") or [] values = [m.group(1) for m in map(_APPLY_UPDATES.match, lines) if m] return bool(values) and values[-1].lower() in _TRUE -def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]: +def _auto_updates(sections: dict[str, list[str]]) -> bool | None: timers = _timer_states(sections) if timers.get(_DNF_INSTALL_TIMER) == "enabled": return True @@ -265,7 +264,7 @@ def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]: return switched_on and installed and timers.get(_APT_TIMER) not in _OFF_STATES -def _interval(sections: Dict[str, List[str]]) -> Optional[int]: +def _interval(sections: dict[str, list[str]]) -> int | None: """apt's interval in seconds, the way ``apt.systemd.daily`` reads it; 0 for "always".""" value = _periodic(sections) if value == "always": @@ -276,19 +275,19 @@ def _interval(sections: Dict[str, List[str]]) -> Optional[int]: return int(match.group(1)) * _UNIT_SECONDS[match.group(2)] -def _last_success(sections: Dict[str, List[str]]) -> Optional[int]: +def _last_success(sections: dict[str, list[str]]) -> int | None: stamp = (sections.get("upgrade-stamp") or [""])[0] return int(stamp) if stamp.isdigit() else None -def _halfway(line: str) -> Optional[InterruptedPackageDict]: +def _halfway(line: str) -> InterruptedPackageDict | None: status, _, name = line.partition(" ") name = name.strip() state = "reinstall-required" if status[2:3] == "R" else _HALFWAY.get(status[1:2]) return {"name": name, "state": state} if name and state else None -def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[InterruptedPackageDict]]: +def _dpkg(sections: dict[str, list[str]]) -> tuple[bool | None, list[InterruptedPackageDict]]: if "dpkg-audit" not in sections or "dpkg-busy" in sections: return None, [] packages = [p for p in map(_halfway, sections["dpkg-audit"]) if p is not None] @@ -296,9 +295,9 @@ def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[Interrup return bool(packages) or journal, packages -def _filesystems(sections: Dict[str, List[str]]) -> Optional[List[FilesystemUsageDict]]: +def _filesystems(sections: dict[str, list[str]]) -> list[FilesystemUsageDict] | None: """Each filesystem df named, once and in its order; None when it named none.""" - found: Dict[str, FilesystemUsageDict] = {} + found: dict[str, FilesystemUsageDict] = {} for line in sections.get("filesystems") or []: match = _DF_LINE.match(line) if not match or int(match.group(1)) == 0: diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index eb42575..8cec25b 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -10,25 +10,26 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin -from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents from napalm_device_types.models import ( HypervisorHostDict, - NICConfigDict, NetworkTargetDict, + NICConfigDict, SnapshotDict, StorageTargetDict, StorageVolumeDict, + VirtualNetworkDict, VMConfigDict, VMCpuTypeDict, VMDict, VMProvisionResultDict, VMStatusDict, - VirtualNetworkDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver): @@ -52,14 +53,13 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri #: the driver can provision; netOrk reads this off the class. GUEST_AGENTS: GuestAgents = {"linux": QEMU_GUEST_AGENTS["linux"]} - # ------------------------------------------------------------------ # Virtual machines – read # ------------------------------------------------------------------ if TYPE_CHECKING: - def get_vms(self) -> List[VMDict]: + def get_vms(self) -> list[VMDict]: """ Returns a list of all virtual machines known to this hypervisor, including their runtime status. @@ -262,7 +262,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Snapshots # ------------------------------------------------------------------ - def get_vm_snapshots(self, name: str) -> List[SnapshotDict]: + def get_vm_snapshots(self, name: str) -> list[SnapshotDict]: """ Returns all snapshots of a virtual machine. @@ -301,8 +301,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def create_vm_snapshot(self, name: str, snapshot: str, - description: str = "", include_memory: bool = False) -> None: + def create_vm_snapshot( + self, name: str, snapshot: str, description: str = "", include_memory: bool = False + ) -> None: """ Creates a snapshot of a virtual machine. @@ -359,7 +360,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Storage # ------------------------------------------------------------------ - def get_vm_storage_pools(self) -> Dict[str, StorageVolumeDict]: + def get_vm_storage_pools(self) -> dict[str, StorageVolumeDict]: """ Returns the storage pools / datastores configured on this hypervisor. @@ -403,7 +404,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Virtual networking # ------------------------------------------------------------------ - def get_virtual_networks(self) -> Dict[str, VirtualNetworkDict]: + def get_virtual_networks(self) -> dict[str, VirtualNetworkDict]: """ Returns the virtual networks / bridges defined on this hypervisor. @@ -443,11 +444,6 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Package management (hypervisor extensions / plugins) # ------------------------------------------------------------------ - - - - - # ------------------------------------------------------------------ # Virtual machine provisioning # ------------------------------------------------------------------ @@ -459,10 +455,10 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri image_url: str, cpu: int, memory: int, - nics: List[NICConfigDict], - cloud_init_config: Dict[str, Any], + nics: list[NICConfigDict], + cloud_init_config: dict[str, Any], image_checksum: str | None = None, - ssh_public_keys: List[str] | None = None, + ssh_public_keys: list[str] | None = None, disk_resize_gb: int | None = None, storage: str | None = None, cpu_type: str | None = None, @@ -602,7 +598,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def get_network_targets(self) -> List[NetworkTargetDict]: + def get_network_targets(self) -> list[NetworkTargetDict]: """ List the network targets a new VM's NIC may attach to. @@ -620,7 +616,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def get_image_storages(self) -> List[StorageTargetDict]: + def get_image_storages(self) -> list[StorageTargetDict]: """ List the storage pools a new VM's root disk may be placed on, scoped to the specific node the VM will be created on. @@ -638,13 +634,13 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def get_hypervisor_hosts(self) -> List[HypervisorHostDict]: + def get_hypervisor_hosts(self) -> list[HypervisorHostDict]: """The hosts this hypervisor or its manager runs, and how each stands: connected, in maintenance, with configuration issues, under a central manager. Facts; what is a problem is the caller's to decide.""" ... - def get_vm_cpu_types(self) -> List[VMCpuTypeDict]: + def get_vm_cpu_types(self) -> list[VMCpuTypeDict]: """ List the virtual CPU models a new VM may be given on the node it will be created on. diff --git a/napalm_device_types/interface_filter.py b/napalm_device_types/interface_filter.py index 4360e58..45599d7 100644 --- a/napalm_device_types/interface_filter.py +++ b/napalm_device_types/interface_filter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Dropping interfaces that are noise rather than signal. An access point exposes radio PHYs and a loopback alongside its real @@ -9,19 +8,19 @@ attributes are the customisation point. from __future__ import annotations -from typing import Any, ClassVar, Dict, FrozenSet, Tuple +from typing import Any, ClassVar class InterfaceFilterMixin: """Adds :meth:`_filter_interfaces` for drivers that report raw interface dicts.""" #: Interface names dropped outright. - _EXCLUDED_INTERFACES: ClassVar[FrozenSet[str]] = frozenset({"lo"}) + _EXCLUDED_INTERFACES: ClassVar[frozenset[str]] = frozenset({"lo"}) #: Name prefixes dropped -- ``phy*`` are radio devices, not links. - _EXCLUDED_INTERFACE_PREFIXES: ClassVar[Tuple[str, ...]] = ("phy",) + _EXCLUDED_INTERFACE_PREFIXES: ClassVar[tuple[str, ...]] = ("phy",) - def _filter_interfaces(self, interfaces: Dict[str, Any]) -> Dict[str, Any]: + def _filter_interfaces(self, interfaces: dict[str, Any]) -> dict[str, Any]: """Remove loopback and radio-device (phy*) interfaces from an interface dict.""" return { name: data diff --git a/napalm_device_types/kernel.py b/napalm_device_types/kernel.py index 463c2c4..bf31cca 100644 --- a/napalm_device_types/kernel.py +++ b/napalm_device_types/kernel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """What the running kernel has built and loaded. A kernel CVE's exploitability often hangs on code that is simply not there -- @@ -25,7 +24,7 @@ import base64 import binascii import gzip import zlib -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import KernelFactsDict @@ -78,9 +77,9 @@ def _report(output: str) -> str: return report -def _sections(report: str) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: Optional[List[str]] = None +def _sections(report: str) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] | None = None for line in report.splitlines(): line = line.strip() if line.startswith("[") and line.endswith("]"): @@ -90,8 +89,8 @@ def _sections(report: str) -> Dict[str, List[str]]: return sections -def _config(lines: List[str]) -> Dict[str, str]: - config: Dict[str, str] = {} +def _config(lines: list[str]) -> dict[str, str]: + config: dict[str, str] = {} for line in lines: option, sep, value = line.partition("=") if not sep: @@ -112,7 +111,7 @@ def parse_kernel_facts(output: str) -> KernelFactsDict: """ sections = _sections(_report(output)) - def names(key: str) -> Optional[List[str]]: + def names(key: str) -> list[str] | None: if key not in sections: return None return sorted({module_name(line) for line in sections[key]}) diff --git a/napalm_device_types/lag.py b/napalm_device_types/lag.py index 1d715d0..3252db2 100644 --- a/napalm_device_types/lag.py +++ b/napalm_device_types/lag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Logical LAG entries for ``get_interfaces()``, built from their member ports. Some switches list only physical ports, each tagged with the trunk it belongs @@ -10,17 +9,17 @@ the same for every vendor, so it lives here once; a driver only has to set from __future__ import annotations import re -from typing import Any, Dict, List, Optional +from typing import Any -def _port_order(name: str) -> List[Any]: +def _port_order(name: str) -> list[Any]: return [int(p) if p.isdigit() else p for p in re.split(r"(\d+)", name)] def add_lag_interfaces( - interfaces: Dict[str, Dict[str, Any]], - lag_modes: Optional[Dict[str, str]] = None, -) -> Dict[str, Dict[str, Any]]: + interfaces: dict[str, dict[str, Any]], + lag_modes: dict[str, str] | None = None, +) -> dict[str, dict[str, Any]]: """Return *interfaces* plus one logical entry per ``trunk_group``. The LAG entry is up/enabled if any member is, its speed is the members' @@ -31,7 +30,7 @@ def add_lag_interfaces( mode gets no ``lag_mode`` key rather than a guessed one. """ result = dict(interfaces) - groups: Dict[str, List[str]] = {} + groups: dict[str, list[str]] = {} for name, iface in interfaces.items(): group = iface.get("trunk_group") if group: @@ -41,7 +40,7 @@ def add_lag_interfaces( if group in result: continue members = sorted(members, key=_port_order) - lag: Dict[str, Any] = { + lag: dict[str, Any] = { "is_up": any(interfaces[m].get("is_up") for m in members), "is_enabled": any(interfaces[m].get("is_enabled") for m in members), "description": f"LAG ({', '.join(members)})", diff --git a/napalm_device_types/licenses.py b/napalm_device_types/licenses.py index c5a6b83..ceeb1ea 100644 --- a/napalm_device_types/licenses.py +++ b/napalm_device_types/licenses.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Licenses and subscriptions a device reports. A fact for the caller (docs/STRUCTURE.md): whether a free edition, a missing @@ -8,7 +7,7 @@ driver. Declared only; a driver that can read licenses implements it. from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import LicenseDict @@ -16,7 +15,7 @@ from napalm_device_types.models import LicenseDict class LicenseMixin: if TYPE_CHECKING: # pragma: no cover - declared for type checkers only - def get_licenses(self) -> List[LicenseDict]: + def get_licenses(self) -> list[LicenseDict]: """Every license or subscription the device knows of, with its status. Example:: diff --git a/napalm_device_types/listening.py b/napalm_device_types/listening.py index 73a625e..a622bd9 100644 --- a/napalm_device_types/listening.py +++ b/napalm_device_types/listening.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Listening sockets: what listens on which address, and which service it is. Whether a service is reachable from outside its host is decided by what it @@ -34,7 +33,7 @@ from __future__ import annotations import re from shlex import quote -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import ListeningSocketDict, ListeningSocketsDict from napalm_device_types.terminal import strip_terminal_codes @@ -58,11 +57,11 @@ LISTENING_SOCKETS_COMMAND = ( "t=netstat; s=$(netstat -lntup 2>&1); r=$?; " "pids=$(printf '%s\\n' \"$s\" | grep -o ' [0-9][0-9]*/' | tr -d ' /'); " "else t=; fi; " - "if [ -n \"$t\" ]; then " - "echo \"[$t]\"; printf '%s\\n' \"$s\"; echo \"__SS_RC=$r\"; " + 'if [ -n "$t" ]; then ' + 'echo "[$t]"; printf \'%s\\n\' "$s"; echo "__SS_RC=$r"; ' "echo '[cgroups]'; " "for p in $(printf '%s\\n' \"$pids\" | sort -u); do " - "sed \"s|^|$p |\" /proc/$p/cgroup 2>/dev/null; done; " + 'sed "s|^|$p |" /proc/$p/cgroup 2>/dev/null; done; ' "else echo '[no-ss]'; fi; " "printf '%s%s\\n' SOCK_ END" ) @@ -88,7 +87,7 @@ class ListeningSocketsUnavailable(NotImplementedError): """The host has neither ``ss`` nor ``netstat``; nothing to read, nothing to retry.""" -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -98,9 +97,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: List[str] = [] +def _sections(lines: list[str]) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines: if line.startswith("[") and line.endswith("]") and " " not in line: current = sections.setdefault(line[1:-1], []) @@ -109,7 +108,7 @@ def _sections(lines: List[str]) -> Dict[str, List[str]]: return sections -def _split_local(local: str) -> Optional[Tuple[str, Optional[str], int]]: +def _split_local(local: str) -> tuple[str, str | None, int] | None: """``[fe80::1]%eth0:546`` (or ``[fe80::1%eth0]:546``) -> ``("fe80::1", "eth0", 546)``; None if no port.""" host, sep, port = local.rpartition(":") @@ -122,9 +121,9 @@ def _split_local(local: str) -> Optional[Tuple[str, Optional[str], int]]: return address or "*", zone or None, int(port) -def _cgroup_paths(lines: List[str]) -> Dict[int, str]: +def _cgroup_paths(lines: list[str]) -> dict[int, str]: """Each process's cgroup path: the unified hierarchy, or systemd's under v1.""" - paths: Dict[int, str] = {} + paths: dict[int, str] = {} for line in lines: pid, _, entry = line.partition(" ") parts = entry.split(":", 2) @@ -136,7 +135,7 @@ def _cgroup_paths(lines: List[str]) -> Dict[int, str]: return paths -def _unit(path: Optional[str]) -> Optional[str]: +def _unit(path: str | None) -> str | None: services = _SERVICE_RE.findall(path or "") if services: return str(services[-1]) @@ -144,17 +143,17 @@ def _unit(path: Optional[str]) -> Optional[str]: return str(procd.group(1)) if procd else None -def _container(path: Optional[str]) -> Optional[str]: +def _container(path: str | None) -> str | None: match = _CONTAINER_RE.search(path or "") return (match.group(1) or match.group(2)) if match else None def _entry( proto: str, - local: Tuple[str, Optional[str], int], - process: Optional[str], - pid: Optional[int], - paths: Dict[int, str], + local: tuple[str, str | None, int], + process: str | None, + pid: int | None, + paths: dict[int, str], ) -> ListeningSocketDict: address, interface, port = local path = paths.get(pid) if pid is not None else None @@ -170,7 +169,7 @@ def _entry( } -def _ss_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: +def _ss_socket(line: str, paths: dict[int, str]) -> ListeningSocketDict | None: parts = line.split() if len(parts) < 5 or parts[0] not in _PROTOCOLS: return None @@ -182,7 +181,7 @@ def _ss_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict return _entry(parts[0], local, process, pid, paths) -def _netstat_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: +def _netstat_socket(line: str, paths: dict[int, str]) -> ListeningSocketDict | None: """``Proto Recv-Q Send-Q Local Foreign [State] PID/Program`` -- a UDP line has no state, and ``-`` is a socket without a process.""" parts = line.split() @@ -204,7 +203,7 @@ def _netstat_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocke _PARSERS = {"ss": _ss_socket, "netstat": _netstat_socket} -def parse_listening_sockets(output: str) -> List[ListeningSocketDict]: +def parse_listening_sockets(output: str) -> list[ListeningSocketDict]: """Parse what :data:`LISTENING_SOCKETS_COMMAND` printed, sorted by protocol, port and address. @@ -223,7 +222,9 @@ def parse_listening_sockets(output: str) -> List[ListeningSocketDict]: paths = _cgroup_paths(sections.get("cgroups", [])) parse = _PARSERS[tool] sockets = [s for s in (parse(line, paths) for line in lines) if s is not None] - return sorted(sockets, key=lambda s: (s["proto"], s["port"], s["address"], s["interface"] or "")) + return sorted( + sockets, key=lambda s: (s["proto"], s["port"], s["address"], s["interface"] or "") + ) class ListeningSocketsMixin: diff --git a/napalm_device_types/mac_acl.py b/napalm_device_types/mac_acl.py index a84e3cb..a72e5d5 100644 --- a/napalm_device_types/mac_acl.py +++ b/napalm_device_types/mac_acl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """MAC-based access control, per SSID on an AP and per port on a switch. The reader is identical either way; only the writer is AP-specific so far. @@ -10,7 +9,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import MACACLDict @@ -18,7 +17,7 @@ from napalm_device_types.models import MACACLDict class MacAclMixin: if TYPE_CHECKING: - def get_mac_acl(self) -> Dict[str, MACACLDict]: + def get_mac_acl(self) -> dict[str, MACACLDict]: """ Returns the MAC-address-based access control lists configured per SSID. @@ -59,7 +58,7 @@ class MacAclMixin: """ ... - def push_mac_acl(self, ssid_name: str, mode: str, macs: List[str]) -> None: + def push_mac_acl(self, ssid_name: str, mode: str, macs: list[str]) -> None: """ Rewrites the MAC-address access control list for a single SSID. diff --git a/napalm_device_types/management_controller.py b/napalm_device_types/management_controller.py index a1d6df4..e9a9de5 100644 --- a/napalm_device_types/management_controller.py +++ b/napalm_device_types/management_controller.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for server baseboard management controllers (BMCs). @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from napalm_device_types.base import DeviceTypeDriver @@ -107,7 +106,7 @@ class ManagementControllerDriver(DeviceTypeDriver): """ ... - def get_event_log(self, limit: int = 500) -> List[EventLogEntryDict]: + def get_event_log(self, limit: int = 500) -> list[EventLogEntryDict]: """ Returns the BMC's system event log (SEL, Lifecycle Log, IML), newest first. @@ -116,7 +115,7 @@ class ManagementControllerDriver(DeviceTypeDriver): """ ... - def get_firmware_inventory(self) -> List[FirmwareDict]: + def get_firmware_inventory(self) -> list[FirmwareDict]: """ Returns every firmware component the BMC reports: its own, the BIOS, storage controllers, NICs, power supplies. diff --git a/napalm_device_types/media.py b/napalm_device_types/media.py index 20c52f9..ce0882e 100644 --- a/napalm_device_types/media.py +++ b/napalm_device_types/media.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for networked media players. @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver @@ -34,7 +33,7 @@ class MediaDriver(DeviceTypeDriver): if TYPE_CHECKING: - def get_playback_state(self) -> Dict[str, Any]: + def get_playback_state(self) -> dict[str, Any]: """ Returns the transport state of the renderer. @@ -51,7 +50,7 @@ class MediaDriver(DeviceTypeDriver): """Sets the output volume, 0-100.""" ... - def get_zone_info(self) -> List[Dict[str, Any]]: + def get_zone_info(self) -> list[dict[str, Any]]: """ Returns the zones or groups this renderer participates in. diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 9ec1641..e2ae9d0 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -5,9 +5,9 @@ These supplement the types defined in napalm.base.models and are used by the abstract device-type driver classes in this package. """ -from typing import Dict, List, Literal, Optional, Tuple, get_args -from typing_extensions import NotRequired, TypedDict +from typing import Literal, get_args +from typing_extensions import NotRequired, TypedDict # --------------------------------------------------------------------------- # Common (shared across device types) @@ -36,7 +36,7 @@ class MACACLDict(TypedDict): name: str policy: str - entries: List[MACACLEntryDict] + entries: list[MACACLEntryDict] class PackageDict(TypedDict): @@ -70,9 +70,9 @@ class UpdateDict(TypedDict): current_version: str new_version: str #: Where the new version comes from, e.g. apt's suites "noble-updates,noble-security". - origin: NotRequired[Optional[str]] + origin: NotRequired[str | None] #: True for a security update, False for a known other one, None when unknown. - security: NotRequired[Optional[bool]] + security: NotRequired[bool | None] class InterruptedPackageDict(TypedDict): @@ -100,24 +100,24 @@ class HostStatusDict(TypedDict): """What a host says about its own patch state (``HostStatusMixin.get_host_status``).""" #: True when the host needs a reboot to finish an update, None when it cannot tell. - reboot_required: Optional[bool] + reboot_required: bool | None #: Why, e.g. "kernel 6.8.0-142-generic installed, 6.8.0-139-generic running". - reboot_reason: Optional[str] + reboot_reason: str | None #: True when the host installs updates on its own (unattended-upgrades, dnf-automatic). - auto_updates: Optional[bool] + auto_updates: bool | None #: True when a package manager run was cut off and has to be finished #: (``dpkg --configure -a``) before anything else installs; None when it #: cannot tell, or a package manager is running right now. - package_manager_interrupted: NotRequired[Optional[bool]] + package_manager_interrupted: NotRequired[bool | None] #: The packages that run left halfway; empty when only its journal says so. - interrupted_packages: NotRequired[List[InterruptedPackageDict]] + interrupted_packages: NotRequired[list[InterruptedPackageDict]] #: When the host's own automatic updates last ran successfully (Unix time). - auto_updates_last_success: NotRequired[Optional[int]] + auto_updates_last_success: NotRequired[int | None] #: How often they are meant to run, in seconds; 0 for every time the timer fires. - auto_updates_interval: NotRequired[Optional[int]] + auto_updates_interval: NotRequired[int | None] #: How full ``/``, ``/boot`` and ``/var`` are, each filesystem once; None when #: it could not be read. - filesystems: NotRequired[Optional[List[FilesystemUsageDict]]] + filesystems: NotRequired[list[FilesystemUsageDict] | None] # --------------------------------------------------------------------------- @@ -163,11 +163,11 @@ class ChannelScanEntryDict(TypedDict): bssid: str ssid: str - frequency: int # centre frequency in MHz, e.g. 2437 - channel: int # channel number, e.g. 6 - signal_dbm: int # RSSI in dBm, e.g. -72 + frequency: int # centre frequency in MHz, e.g. 2437 + channel: int # channel number, e.g. 6 + signal_dbm: int # RSSI in dBm, e.g. -72 channel_width: int # MHz: 20 / 40 / 80 / 160; 0 if unknown - band: str # "2.4GHz" | "5GHz" | "6GHz" + band: str # "2.4GHz" | "5GHz" | "6GHz" class WirelessConfigDict(TypedDict): @@ -226,7 +226,7 @@ class Dot1XConfigDict(TypedDict): enabled: bool ssid: str auth_server: RadiusServerDict - acct_server: Optional[RadiusServerDict] + acct_server: RadiusServerDict | None reauth_interval: int pmksa_caching: bool @@ -263,7 +263,7 @@ class InterfaceConfigDict(TypedDict, total=False): mode: str access_vlan: int voice_vlan: int - trunk_vlans: List[int] + trunk_vlans: list[int] native_vlan: int @@ -280,7 +280,7 @@ class VlanConfigDict(TypedDict, total=False): name: str active: bool - interfaces: List[str] + interfaces: list[str] class STPInterfaceDict(TypedDict): @@ -297,11 +297,11 @@ class SpanningTreeDict(TypedDict): root_priority: int bridge_id: str bridge_priority: int - interfaces: Dict[str, STPInterfaceDict] + interfaces: dict[str, STPInterfaceDict] class PortChannelDict(TypedDict): - members: List[str] + members: list[str] protocol: str min_links: int is_up: bool @@ -314,7 +314,7 @@ class Dot1XPortDict(TypedDict): port_control: str host_mode: str auth_server: RadiusServerDict - acct_server: Optional[RadiusServerDict] + acct_server: RadiusServerDict | None reauthentication: bool reauth_interval: int guest_vlan: int @@ -334,7 +334,7 @@ class PoEPortDict(TypedDict): class PoESummaryDict(TypedDict): total_power_budget: float total_power_draw: float - ports: Dict[str, PoEPortDict] + ports: dict[str, PoEPortDict] # --------------------------------------------------------------------------- @@ -363,10 +363,10 @@ class KernelFactsDict(TypedDict): """ release: str # uname -r - loaded: Optional[List[str]] # /proc/modules - builtin: Optional[List[str]] # modules.builtin -- compiled into the kernel image - available: Optional[List[str]] # modules.dep -- shipped as loadable modules - config: Optional[Dict[str, str]] # build configuration, set options only; quotes stripped + loaded: list[str] | None # /proc/modules + builtin: list[str] | None # modules.builtin -- compiled into the kernel image + available: list[str] | None # modules.dep -- shipped as loadable modules + config: dict[str, str] | None # build configuration, set options only; quotes stripped class ListeningSocketDict(TypedDict): @@ -381,11 +381,11 @@ class ListeningSocketDict(TypedDict): proto: str # "tcp" or "udp" address: str # "0.0.0.0", "::", "*", "127.0.0.1", "::ffff:127.0.0.1", ... port: int - interface: Optional[str] # the %zone a socket is bound to ("lo", "eth0"), if any - process: Optional[str] # the first process holding it; None without one or without root - pid: Optional[int] - unit: Optional[str] # the process's systemd service, without ".service" - container_id: Optional[str] # the full container ID, for a container on the host network + interface: str | None # the %zone a socket is bound to ("lo", "eth0"), if any + process: str | None # the first process holding it; None without one or without root + pid: int | None + unit: str | None # the process's systemd service, without ".service" + container_id: str | None # the full container ID, for a container on the host network class ListeningSocketsDict(TypedDict): @@ -397,7 +397,7 @@ class ListeningSocketsDict(TypedDict): """ attributed: bool - sockets: List[ListeningSocketDict] + sockets: list[ListeningSocketDict] class NetfilterMatchDict(TypedDict): @@ -413,7 +413,7 @@ class NetfilterMatchDict(TypedDict): field: str op: str # "==" or "!=" - values: List[str] + values: list[str] class NetfilterRuleDict(TypedDict): @@ -424,9 +424,9 @@ class NetfilterRuleDict(TypedDict): marks), or ``other`` -- NAT, a queue, a verdict map -- with ``target`` naming it. """ - matches: List[NetfilterMatchDict] + matches: list[NetfilterMatchDict] verdict: str - target: Optional[str] + target: str | None text: str # the rule as iptables-save printed it, or as one nft-style line @@ -437,11 +437,11 @@ class NetfilterChainDict(TypedDict): family: str # "ip", "ip6", "inet", "bridge", "arp", "netdev" table: str name: str - type: Optional[str] # "filter", "nat", "route" - hook: Optional[str] # "prerouting", "input", "forward", "output", "postrouting", ... - priority: Optional[int] - policy: Optional[str] # "accept" or "drop" - rules: List[NetfilterRuleDict] + type: str | None # "filter", "nat", "route" + hook: str | None # "prerouting", "input", "forward", "output", "postrouting", ... + priority: int | None + policy: str | None # "accept" or "drop" + rules: list[NetfilterRuleDict] class HostFirewallDict(TypedDict): @@ -452,8 +452,8 @@ class HostFirewallDict(TypedDict): extensions opaquely) and ``iptables-legacy`` (a ruleset of its own). """ - sources: List[str] - chains: List[NetfilterChainDict] + sources: list[str] + chains: list[NetfilterChainDict] class VirtualIpDict(TypedDict): @@ -471,14 +471,14 @@ class VirtualIpDict(TypedDict): #: CARP address's interface description instance: str address: str # normalised: "10.7.224.10", "fd00::10" - prefix_length: Optional[int] # None where the configuration gives none - interface: Optional[str] + prefix_length: int | None # None where the configuration gives none + interface: str | None #: vip-manager's trigger-key, "vrid:" for keepalived, "vhid:" for CARP - group_key: Optional[str] - running: Optional[bool] # the unit is active; for CARP: in MASTER or BACKUP + group_key: str | None + running: bool | None # the unit is active; for CARP: in MASTER or BACKUP #: The host holds the address now, by the mechanism's own state (CARP's #: MASTER). Absent or None: the interface addresses tell. - holds: NotRequired[Optional[bool]] + holds: NotRequired[bool | None] class ServiceClusterMemberDict(TypedDict): @@ -492,19 +492,19 @@ class ServiceClusterMemberDict(TypedDict): endpoint. """ - name: Optional[str] - address: Optional[str] - port: Optional[int] + name: str | None + address: str | None + port: int | None #: as the mechanism words it: Patroni's "leader"/"sync_standby"/"replica", #: adguardhome-sync's "origin"/"replica", BunkerWeb's "manager"/"worker" - role: Optional[str] + role: str | None #: the member's state where the mechanism reports one (Patroni's "running", #: "streaming"); configuration alone has none - state: Optional[str] + state: str | None this_host: bool #: the index in ``members`` of the member whose configuration this one takes: #: a replica's origin, a worker's manager. None where it takes none. - config_from: Optional[int] + config_from: int | None class ServiceClusterDict(TypedDict): @@ -525,9 +525,9 @@ class ServiceClusterDict(TypedDict): #: what the members share where the reading host can name it: Patroni's scope, #: the sync's origin ("10.9.100.5:3000"), a worker's manager address. None for a #: manager, which cannot name its own address. - cluster: Optional[str] - running: Optional[bool] # the unit is active; a listed container runs - members: List[ServiceClusterMemberDict] + cluster: str | None + running: bool | None # the unit is active; a listed container runs + members: list[ServiceClusterMemberDict] class ServiceClustersDict(TypedDict): @@ -538,9 +538,9 @@ class ServiceClustersDict(TypedDict): container engine it may not ask, a sync configured by file. """ - patroni: Optional[List[ServiceClusterDict]] - adguardhome_sync: Optional[List[ServiceClusterDict]] - bunkerweb: Optional[List[ServiceClusterDict]] + patroni: list[ServiceClusterDict] | None + adguardhome_sync: list[ServiceClusterDict] | None + bunkerweb: list[ServiceClusterDict] | None class VirtualIpsDict(TypedDict): @@ -553,11 +553,11 @@ class VirtualIpsDict(TypedDict): keeps what it knew for a None, and for a missing ``carp``. """ - vip_manager: Optional[List[VirtualIpDict]] - keepalived: Optional[List[VirtualIpDict]] + vip_manager: list[VirtualIpDict] | None + keepalived: list[VirtualIpDict] | None #: A firewall's CARP addresses (not read by ``VirtualIpsMixin``, which reads Linux #: hosts): every address bound to a VHID, the CARP address and IP aliases on it. - carp: NotRequired[Optional[List[VirtualIpDict]]] + carp: NotRequired[list[VirtualIpDict] | None] class PortForwardDict(TypedDict): @@ -576,7 +576,7 @@ class PortForwardDict(TypedDict): class SecurityZoneDict(TypedDict): - interfaces: List[str] + interfaces: list[str] policy: str description: str @@ -622,12 +622,12 @@ class FirewallRuleDict(TypedDict): class FirewallRuleUpdateDict(TypedDict): uuid: str rule: FirewallRuleDict - changed_fields: List[str] + changed_fields: list[str] class FirewallRuleDiffDict(TypedDict): - add: List[FirewallRuleDict] - update: List[FirewallRuleUpdateDict] + add: list[FirewallRuleDict] + update: list[FirewallRuleUpdateDict] class DhcpReservationDict(TypedDict): @@ -656,12 +656,12 @@ class DhcpReservationDict(TypedDict): class DhcpReservationUpdateDict(TypedDict): uuid: str reservation: DhcpReservationDict - changed_fields: List[str] + changed_fields: list[str] class DhcpReservationDiffDict(TypedDict): - add: List[DhcpReservationDict] - update: List[DhcpReservationUpdateDict] + add: list[DhcpReservationDict] + update: list[DhcpReservationUpdateDict] class DhcpOptionDataDict(TypedDict, total=False): @@ -676,11 +676,11 @@ class DhcpOptionDataDict(TypedDict, total=False): desired value is empty". A driver must preserve options it was not given. """ - routers: List[str] - domain_name_servers: List[str] + routers: list[str] + domain_name_servers: list[str] domain_name: str - domain_search: List[str] - ntp_servers: List[str] + domain_search: list[str] + ntp_servers: list[str] class DhcpSubnetDict(TypedDict): @@ -702,7 +702,7 @@ class DhcpSubnetDict(TypedDict): uuid: str subnet: str description: str - pools: List[str] + pools: list[str] option_data: DhcpOptionDataDict match_client_id: bool @@ -710,12 +710,12 @@ class DhcpSubnetDict(TypedDict): class DhcpSubnetUpdateDict(TypedDict): uuid: str subnet: DhcpSubnetDict - changed_fields: List[str] + changed_fields: list[str] class DhcpSubnetDiffDict(TypedDict): - add: List[DhcpSubnetDict] - update: List[DhcpSubnetUpdateDict] + add: list[DhcpSubnetDict] + update: list[DhcpSubnetUpdateDict] class DhcpHaPeerDict(TypedDict): @@ -742,7 +742,7 @@ class DhcpHaDict(TypedDict): enabled: bool #: Which of ``peers`` is this server. this_server: NotRequired[str] - peers: List[DhcpHaPeerDict] + peers: list[DhcpHaPeerDict] class ConfigSyncDict(TypedDict): @@ -752,12 +752,12 @@ class ConfigSyncDict(TypedDict): host without the feature switched on). """ - peer: Optional[str] + peer: str | None #: The services whose configuration it pushes, by the name ``get_services`` #: reports. - services: List[str] + services: list[str] #: Sync items the driver has no service name for, as the host names them. - unmapped: List[str] + unmapped: list[str] class VPNTunnelDict(TypedDict): @@ -834,8 +834,8 @@ class VMDict(TypedDict): class VMPassthroughDict(TypedDict): """A host device handed through to a VM (PCI, USB).""" - slot: str # hypervisor's device key, e.g. "hostpci0" - kind: str # "pci" or "usb" + slot: str # hypervisor's device key, e.g. "hostpci0" + kind: str # "pci" or "usb" config: str # hypervisor's own description of the device @@ -845,19 +845,19 @@ class VMConfigDict(TypedDict): vcpus: int memory: int os_type: str - boot_order: List[str] - disks: List[VMDiskDict] - nics: List[VMNICDict] + boot_order: list[str] + disks: list[VMDiskDict] + nics: list[VMNICDict] description: str - tags: List[str] + tags: list[str] # Hardware details not every hypervisor exposes; absent when unknown. - os_name: NotRequired[str] # human-readable guest OS, e.g. "Ubuntu Linux (64-bit)" - cpu_type: NotRequired[str] # e.g. "host", "kvm64" + os_name: NotRequired[str] # human-readable guest OS, e.g. "Ubuntu Linux (64-bit)" + cpu_type: NotRequired[str] # e.g. "host", "kvm64" sockets: NotRequired[int] cores_per_socket: NotRequired[int] - firmware: NotRequired[str] # "bios" or "efi" - machine: NotRequired[str] # machine type / virtual hardware version - passthrough: NotRequired[List[VMPassthroughDict]] + firmware: NotRequired[str] # "bios" or "efi" + machine: NotRequired[str] # machine type / virtual hardware version + passthrough: NotRequired[list[VMPassthroughDict]] class StorageVolumeDict(TypedDict): @@ -900,29 +900,29 @@ class PhysicalDiskDict(TypedDict): model: str serial: str vendor: str - type: str # "hdd", "ssd", "nvme" - size: int # bytes - rpm: int # 0 for SSD/NVMe - temperature: int # Celsius; -1 if unavailable - health: str # "healthy", "warning", "failed", "unknown" - pool: str # name of the containing pool; empty string if spare/unassigned + type: str # "hdd", "ssd", "nvme" + size: int # bytes + rpm: int # 0 for SSD/NVMe + temperature: int # Celsius; -1 if unavailable + health: str # "healthy", "warning", "failed", "unknown" + pool: str # name of the containing pool; empty string if spare/unassigned class DiskPoolDict(TypedDict): """A RAID array, ZFS pool, or volume group.""" name: str - type: str # "zfs", "lvm", "md", "hardware-raid", "btrfs" - level: str # "stripe", "mirror", "raidz1", "raidz2", "raidz3", - # "raid0" … "raid60", "single", etc. - status: str # "online", "degraded", "faulted", "offline", "unknown" - total: int # bytes - used: int # bytes - available: int # bytes - disks: List[str] # slot identifiers of member disks + type: str # "zfs", "lvm", "md", "hardware-raid", "btrfs" + level: str # "stripe", "mirror", "raidz1", "raidz2", "raidz3", + # "raid0" … "raid60", "single", etc. + status: str # "online", "degraded", "faulted", "offline", "unknown" + total: int # bytes + used: int # bytes + available: int # bytes + disks: list[str] # slot identifiers of member disks auto_expand: bool dedup: bool - compression: str # "off", "lz4", "gzip", "zstd", etc. + compression: str # "off", "lz4", "gzip", "zstd", etc. class LogicalVolumeDict(TypedDict): @@ -930,74 +930,74 @@ class LogicalVolumeDict(TypedDict): name: str pool: str - type: str # "filesystem", "volume" (block device / LUN), "zvol" - total: int # bytes - used: int # bytes - available: int # bytes - mountpoint: str # empty string for block volumes - compression: str # "off", "lz4", etc. + type: str # "filesystem", "volume" (block device / LUN), "zvol" + total: int # bytes + used: int # bytes + available: int # bytes + mountpoint: str # empty string for block volumes + compression: str # "off", "lz4", etc. dedup: bool readonly: bool - snapshots: int # number of snapshots currently held + snapshots: int # number of snapshots currently held class NASShareDict(TypedDict): """A network share or iSCSI target exported by the storage device.""" name: str - protocol: str # "nfs", "smb", "afp", "ftp", "sftp", "iscsi", "webdav" - path: str # local filesystem path or iSCSI target IQN - volume: str # logical volume or dataset this share is backed by + protocol: str # "nfs", "smb", "afp", "ftp", "sftp", "iscsi", "webdav" + path: str # local filesystem path or iSCSI target IQN + volume: str # logical volume or dataset this share is backed by enabled: bool readonly: bool description: str - clients: List[str] # IP/subnet allow-list; empty list = all hosts allowed + clients: list[str] # IP/subnet allow-list; empty list = all hosts allowed class VolumeSnapshotDict(TypedDict): """A point-in-time snapshot of a logical volume or dataset.""" name: str - volume: str # parent volume / dataset - created: float # Unix epoch - size: int # bytes of unique data referenced by this snapshot + volume: str # parent volume / dataset + created: float # Unix epoch + size: int # bytes of unique data referenced by this snapshot description: str - clones: List[str] # volumes cloned from this snapshot + clones: list[str] # volumes cloned from this snapshot class StorageQuotaDict(TypedDict): """A filesystem quota applied to a user, group, or dataset.""" - target: str # username, group name, or dataset path - target_type: str # "user", "group", "dataset" - volume: str # volume / dataset the quota applies to - used: int # bytes currently used - quota: int # hard limit in bytes; 0 = no limit - ref_quota: int # referenced-data limit in bytes; 0 = no limit + target: str # username, group name, or dataset path + target_type: str # "user", "group", "dataset" + volume: str # volume / dataset the quota applies to + used: int # bytes currently used + quota: int # hard limit in bytes; 0 = no limit + ref_quota: int # referenced-data limit in bytes; 0 = no limit class StorageServiceDict(TypedDict): """Status of a file-sharing or access service running on the device.""" - name: str # "nfs", "smb", "ftp", "ssh", "iscsi", "webdav", etc. - enabled: bool # administratively enabled - running: bool # currently active / listening - port: int # primary listening port; 0 if N/A - version: str # protocol version string (e.g. "4.1" for NFSv4.1) + name: str # "nfs", "smb", "ftp", "ssh", "iscsi", "webdav", etc. + enabled: bool # administratively enabled + running: bool # currently active / listening + port: int # primary listening port; 0 if N/A + version: str # protocol version string (e.g. "4.1" for NFSv4.1) class ReplicationJobDict(TypedDict): """A scheduled replication or sync task.""" name: str - source: str # source path / dataset - target: str # destination path / dataset (may be remote: host:path) - direction: str # "push" or "pull" - schedule: str # cron expression or human label ("daily", "hourly") + source: str # source path / dataset + target: str # destination path / dataset (may be remote: host:path) + direction: str # "push" or "pull" + schedule: str # cron expression or human label ("daily", "hourly") enabled: bool - last_run: float # Unix epoch; 0.0 if never run - last_status: str # "success", "failed", "running", "pending" - bytes_sent: int # bytes transferred in the last run + last_run: float # Unix epoch; 0.0 if never run + last_status: str # "success", "failed", "running", "pending" + bytes_sent: int # bytes transferred in the last run # --------------------------------------------------------------------------- @@ -1013,7 +1013,7 @@ class UserDict(TypedDict): gid: int home: str shell: str - groups: List[str] # all supplementary group names + groups: list[str] # all supplementary group names class ProcessDict(TypedDict): @@ -1022,30 +1022,30 @@ class ProcessDict(TypedDict): pid: int ppid: int user: str - cpu: float # CPU utilisation 0.0–100.0 - memory: float # RSS percentage of total RAM 0.0–100.0 - vsz: int # virtual memory size in KiB - rss: int # resident set size in KiB - tty: str # controlling terminal; empty string if none - state: str # "R" running, "S" sleeping, "D" uninterruptible, "Z" zombie, etc. - started: str # start time string as reported by ``ps`` (e.g. "12:34" or "May28") - command: str # full command line + cpu: float # CPU utilisation 0.0–100.0 + memory: float # RSS percentage of total RAM 0.0–100.0 + vsz: int # virtual memory size in KiB + rss: int # resident set size in KiB + tty: str # controlling terminal; empty string if none + state: str # "R" running, "S" sleeping, "D" uninterruptible, "Z" zombie, etc. + started: str # start time string as reported by ``ps`` (e.g. "12:34" or "May28") + command: str # full command line class CronJobDict(TypedDict): """A scheduled cron task.""" - user: str # owner of the crontab entry - schedule: str # five-field cron expression, e.g. "0 * * * *" - command: str # shell command to execute + user: str # owner of the crontab entry + schedule: str # five-field cron expression, e.g. "0 * * * *" + command: str # shell command to execute description: NotRequired[str] # optional inline comment class ApplyUpdatesResultDict(TypedDict): """Result of an ``apply_updates()`` call.""" - success: bool # True if the package manager exited without error - output: str # raw stdout/stderr captured from the package manager + success: bool # True if the package manager exited without error + output: str # raw stdout/stderr captured from the package manager error: NotRequired[str] # error message if success is False @@ -1056,24 +1056,24 @@ class ContainerEngineDict(TypedDict): it is read and modelled above the driver, in netOrk (netork#765). """ - engine: str # "docker" - api: str # "docker-engine": the Docker Engine API over ``system dial-stdio`` + engine: str # "docker" + api: str # "docker-engine": the Docker Engine API over ``system dial-stdio`` class DeviceActionResultDict(TypedDict): """Return value of ``run_device_action()``.""" - success: bool # True if the action completed without error - output: str # human-readable output or status message + success: bool # True if the action completed without error + output: str # human-readable output or status message class SNMPConfigDict(TypedDict): """SNMP agent configuration detected on the device.""" - running: bool # True if the SNMP daemon is active - community: str # read community string (e.g. "public") - port: int # listening port (default 161) - version: str # highest supported version: "1", "2c", or "3" + running: bool # True if the SNMP daemon is active + community: str # read community string (e.g. "public") + port: int # listening port (default 161) + version: str # highest supported version: "1", "2c", or "3" # --------------------------------------------------------------------------- @@ -1089,7 +1089,7 @@ class HealthMetricsIfaceDict(TypedDict): tx_bytes: NotRequired[int] rx_errors: NotRequired[int] tx_errors: NotRequired[int] - tx_drops: NotRequired[int] # populated instead of tx_errors when the driver reports drops + tx_drops: NotRequired[int] # populated instead of tx_errors when the driver reports drops speed_mbps: NotRequired[int] @@ -1108,7 +1108,7 @@ class HealthMetricsDict(TypedDict): load_1: NotRequired[float] load_5: NotRequired[float] load_15: NotRequired[float] - interfaces: NotRequired[Dict[str, HealthMetricsIfaceDict]] + interfaces: NotRequired[dict[str, HealthMetricsIfaceDict]] LicenseStatus = Literal["active", "missing", "expired", "invalid", "none", "unknown"] @@ -1124,13 +1124,13 @@ class LicenseDict(TypedDict): name: str # what it licenses, as the device names it status: LicenseStatus # "none": the device runs without one, as installed edition: NotRequired[str] # the edition or level, e.g. "Free", "Enterprise Plus" - expires: NotRequired[Optional[str]] # ISO date, None for no expiry + expires: NotRequired[str | None] # ISO date, None for no expiry detail: NotRequired[str] # the device's own words, e.g. why a feature is refused # What the device withholds without it while the rest works, e.g. ["drives", # "firmware inventory"] on a BMC that answers but wants a further license. # Absent: the device says nothing of the kind, or withholds everything it # would offer with it. - withholds: NotRequired[List[str]] + withholds: NotRequired[list[str]] UpdateState = Literal["idle", "running", "done", "failed"] @@ -1170,9 +1170,9 @@ class OspfInterfaceDict(TypedDict): class OspfDict(TypedDict): """The device's OSPF state (``OspfMixin.get_ospf``).""" - router_id: Optional[str] - areas: List[str] - interfaces: Dict[str, OspfInterfaceDict] # by interface name + router_id: str | None + areas: list[str] + interfaces: dict[str, OspfInterfaceDict] # by interface name class FirmwareOfferDict(TypedDict): @@ -1189,10 +1189,10 @@ class HypervisorHostDict(TypedDict): """One host of a hypervisor or cluster (``HypervisorDriver.get_hypervisor_hosts``).""" name: str - connected: Optional[bool] # reachable by its manager; None where it cannot tell - maintenance: Optional[bool] # in maintenance mode - config_issues: List[str] # the manager's configuration warnings, verbatim - managed_by: NotRequired[Optional[str]] # the address of a central manager, if any + connected: bool | None # reachable by its manager; None where it cannot tell + maintenance: bool | None # in maintenance mode + config_issues: list[str] # the manager's configuration warnings, verbatim + managed_by: NotRequired[str | None] # the address of a central manager, if any class FixStepOutcomeDict(TypedDict): @@ -1214,7 +1214,7 @@ class FixResultDict(TypedDict): """The result of a fix routine: every step that ran, up to the first failure.""" success: bool - steps: List[FixStepDict] + steps: list[FixStepDict] output: str # the steps' output, one "[step] output" line each @@ -1222,7 +1222,9 @@ class SnmpAgentSettingsDict(TypedDict): """What the caller wants an SNMP agent to be (``SnmpAgentMixin``).""" community: str # the read-only v2c community - allowed_sources: List[str] # addresses or CIDRs let through to the agent; [] changes no firewall + allowed_sources: list[ + str + ] # addresses or CIDRs let through to the agent; [] changes no firewall contact: NotRequired[str] # sysContact location: NotRequired[str] # sysLocation listen: NotRequired[str] # the address the agent binds to; absent: every address @@ -1237,7 +1239,7 @@ class SnmpValueSpec(TypedDict, total=False): with text. """ - oids: List[str] + oids: list[str] table: str fold: Literal["first", "average", "max"] pattern: str @@ -1346,7 +1348,7 @@ class VMCpuTypeDict(TypedDict): name: str # Model name, usable directly as create_vm_from_cloud_init(cpu_type=...) description: str # One line on what the model is for, for a picker - features: List[str] # cpuinfo flags the guest is guaranteed to see + features: list[str] # cpuinfo flags the guest is guaranteed to see available: bool # False when this node's CPU cannot run the model default: bool # The model create_vm_from_cloud_init uses when cpu_type is None @@ -1359,19 +1361,19 @@ class VMCpuTypeDict(TypedDict): class PingSweepEntryDict(TypedDict): """Outcome of a single ``ping`` inside a sweep (see ``PingSweepMixin``).""" - ip: str # destination that was probed - alive: bool # True if at least one probe was answered - rtt_ms: Optional[float] # average round-trip time in ms; None if unreachable + ip: str # destination that was probed + alive: bool # True if at least one probe was answered + rtt_ms: float | None # average round-trip time in ms; None if unreachable error: NotRequired[str] # driver/transport error for this destination class PingSweepResultDict(TypedDict): """Result of a ``ping_sweep()`` call.""" - entries: List[PingSweepEntryDict] # one entry per probed destination, in input order - scanned: int # destinations actually probed - alive_count: int # entries with alive=True - truncated: bool # True if targets were dropped at the sweep cap + entries: list[PingSweepEntryDict] # one entry per probed destination, in input order + scanned: int # destinations actually probed + alive_count: int # entries with alive=True + truncated: bool # True if targets were dropped at the sweep cap # --------------------------------------------------------------------------- @@ -1387,29 +1389,29 @@ PowerAction = Literal[ "ForceRestart", "PowerCycle", ] -POWER_ACTIONS: Tuple[str, ...] = get_args(PowerAction) +POWER_ACTIONS: tuple[str, ...] = get_args(PowerAction) class PowerStateDict(TypedDict): """The managed server's power, as its BMC reports it.""" - state: str # "On", "Off", "PoweringOn" or "PoweringOff" - actions: List[str] # the PowerAction values this BMC accepts now + state: str # "On", "Off", "PoweringOn" or "PoweringOff" + actions: list[str] # the PowerAction values this BMC accepts now class BmcCpuDict(TypedDict): - socket: str # "CPU1", "Proc 1" - model: str # "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz" - cores: Optional[int] - threads: Optional[int] - status: str # "OK", "Warning", "Critical", "Absent" + socket: str # "CPU1", "Proc 1" + model: str # "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz" + cores: int | None + threads: int | None + status: str # "OK", "Warning", "Critical", "Absent" class BmcDimmDict(TypedDict): - slot: str # "DIMMA1", "PROC 1 DIMM 3" - size_bytes: Optional[int] - type: str # "DDR4", "DDR5"; empty when unknown - speed_mts: Optional[int] # MT/s + slot: str # "DIMMA1", "PROC 1 DIMM 3" + size_bytes: int | None + type: str # "DDR4", "DDR5"; empty when unknown + speed_mts: int | None # MT/s part_number: str serial_number: str status: str @@ -1419,9 +1421,9 @@ class BmcDriveDict(TypedDict): slot: str model: str serial_number: str - size_bytes: Optional[int] - media: str # "HDD", "SSD"; empty when unknown - protocol: str # "SATA", "SAS", "NVMe"; empty when unknown + size_bytes: int | None + media: str # "HDD", "SSD"; empty when unknown + protocol: str # "SATA", "SAS", "NVMe"; empty when unknown status: str @@ -1429,17 +1431,17 @@ class BmcPsuDict(TypedDict): slot: str model: str serial_number: str - capacity_watts: Optional[int] + capacity_watts: int | None status: str class BmcNicDict(TypedDict): """A network port of the managed server, not of the BMC.""" - name: str # "NIC.Integrated.1-1", "Port 1" - mac_address: str # lowercase, colon-separated; empty when unknown - speed_mbps: Optional[int] - link_up: Optional[bool] + name: str # "NIC.Integrated.1-1", "Port 1" + mac_address: str # lowercase, colon-separated; empty when unknown + speed_mbps: int | None + link_up: bool | None class HardwareInventoryDict(TypedDict): @@ -1447,35 +1449,35 @@ class HardwareInventoryDict(TypedDict): manufacturer: str model: str - serial_number: str # the server's, not the BMC's - system_uuid: str # SMBIOS system UUID, lowercase - cpus: List[BmcCpuDict] - memory: List[BmcDimmDict] - drives: List[BmcDriveDict] - power_supplies: List[BmcPsuDict] - nics: List[BmcNicDict] + serial_number: str # the server's, not the BMC's + system_uuid: str # SMBIOS system UUID, lowercase + cpus: list[BmcCpuDict] + memory: list[BmcDimmDict] + drives: list[BmcDriveDict] + power_supplies: list[BmcPsuDict] + nics: list[BmcNicDict] class EventLogEntryDict(TypedDict): """One entry of the BMC's system event log (SEL / Lifecycle / IML).""" id: str - timestamp: Optional[float] # Unix time; None when the BMC gave none - severity: str # "OK", "Warning" or "Critical" + timestamp: float | None # Unix time; None when the BMC gave none + severity: str # "OK", "Warning" or "Critical" message: str class FirmwareDict(TypedDict): """One firmware component: the BMC's own, BIOS, a controller, a NIC.""" - name: str # "BMC", "BIOS", "PERC H755 Front" + name: str # "BMC", "BIOS", "PERC H755 Front" version: str - updateable: Optional[bool] + updateable: bool | None class ProtocolStateDict(TypedDict): enabled: bool - port: Optional[int] # None for a protocol without a port of its own + port: int | None # None for a protocol without a port of its own class ManagementProtocolsDict(TypedDict): @@ -1495,6 +1497,6 @@ class SystemIdentityDict(TypedDict): Each value is empty when unknown or a vendor placeholder.""" - system_uuid: str # lowercase + system_uuid: str # lowercase product_serial: str board_serial: str diff --git a/napalm_device_types/nat_vpn.py b/napalm_device_types/nat_vpn.py index 5d8328a..e85b171 100644 --- a/napalm_device_types/nat_vpn.py +++ b/napalm_device_types/nat_vpn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Address translation and VPN tunnels. A home gateway does a subset of what a firewall does, and these readers are @@ -11,7 +10,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import NATTranslationDict, PortForwardDict, VPNTunnelDict @@ -19,7 +18,7 @@ from napalm_device_types.models import NATTranslationDict, PortForwardDict, VPNT class NatVpnMixin: if TYPE_CHECKING: - def get_nat_translations(self) -> List[NATTranslationDict]: + def get_nat_translations(self) -> list[NATTranslationDict]: """ Returns a list of active NAT translation entries. @@ -47,7 +46,7 @@ class NatVpnMixin: """ ... - def get_port_forwards(self) -> List[PortForwardDict]: + def get_port_forwards(self) -> list[PortForwardDict]: """ Returns the port forwards that let traffic in from the WAN. @@ -87,7 +86,7 @@ class NatVpnMixin: """ ... - def get_vpn_tunnels(self) -> Dict[str, VPNTunnelDict]: + def get_vpn_tunnels(self) -> dict[str, VPNTunnelDict]: """ Returns the status of VPN tunnels. diff --git a/napalm_device_types/os.py b/napalm_device_types/os.py index 27f7287..8c62e09 100644 --- a/napalm_device_types/os.py +++ b/napalm_device_types/os.py @@ -10,11 +10,9 @@ Usage:: ... """ -from typing import List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( CronJobDict, @@ -23,9 +21,14 @@ from napalm_device_types.models import ( SNMPConfigDict, UserDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.updates import UpdateMixin -class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver): +class OSDriver( + UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver +): """ Abstract intermediate driver for general-purpose operating systems (e.g. Linux, BSD, macOS). @@ -44,26 +47,20 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Interfaces matching this pattern are excluded from health-metric collection. # Set True for drivers where the out-error counter actually reports drops (e.g. OPNsense). - # ------------------------------------------------------------------ # Package management # ------------------------------------------------------------------ if TYPE_CHECKING: - - - - # ------------------------------------------------------------------ # Service management # ------------------------------------------------------------------ - # ------------------------------------------------------------------ # Users # ------------------------------------------------------------------ - def get_os_users(self) -> List[UserDict]: + def get_os_users(self) -> list[UserDict]: """ Returns a list of local OS user accounts. @@ -95,7 +92,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Processes # ------------------------------------------------------------------ - def get_processes(self) -> List[ProcessDict]: + def get_processes(self) -> list[ProcessDict]: """ Returns a snapshot of currently running processes. @@ -138,7 +135,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Cron jobs # ------------------------------------------------------------------ - def get_cron_jobs(self) -> List[CronJobDict]: + def get_cron_jobs(self) -> list[CronJobDict]: """ Returns scheduled cron tasks from all user crontabs and ``/etc/cron.d``. @@ -166,7 +163,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # SNMP # ------------------------------------------------------------------ - def get_snmp_config(self) -> Optional[SNMPConfigDict]: + def get_snmp_config(self) -> SNMPConfigDict | None: """ Returns the SNMP agent configuration currently active on the device, or ``None`` if no SNMP daemon is running or detectable. diff --git a/napalm_device_types/ospf.py b/napalm_device_types/ospf.py index 33a96fe..72be557 100644 --- a/napalm_device_types/ospf.py +++ b/napalm_device_types/ospf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """A device's OSPF state, in neutral terms. The routing daemon's own output (FRR, Quagga, a vendor's CLI) is the driver's diff --git a/napalm_device_types/package_updates.py b/napalm_device_types/package_updates.py index 55e3542..7ff7f9f 100644 --- a/napalm_device_types/package_updates.py +++ b/napalm_device_types/package_updates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Pending package updates: which package, from where, and whether it is a security fix. A patch deadline -- "security updates within 14 days" -- needs to know which @@ -16,7 +15,6 @@ security -- netOrk's CVE matching is what catches those. from __future__ import annotations import re -from typing import Dict, List, Set from napalm_device_types.models import UpdateDict from napalm_device_types.terminal import strip_terminal_codes @@ -42,10 +40,10 @@ _APT_STATUS = re.compile(r"^__APT_RC=(\d+)\s*$", re.MULTILINE) _DNF_STATUS = re.compile(r"^__DNF_RC=(\d+)\s*$", re.MULTILINE) -def _joined_lines(output: str) -> List[str]: +def _joined_lines(output: str) -> list[str]: """Lines as apt printed them: a terminal wraps long ones, and the continuation starts with a space.""" - lines: List[str] = [] + lines: list[str] = [] for line in output.splitlines(): if line.startswith(" ") and lines: lines[-1] += line[1:].rstrip() @@ -67,7 +65,7 @@ def _listing(output: str, status: re.Pattern[str], what: str) -> str: return status.sub("", text) -def parse_apt_upgradable(output: str) -> List[UpdateDict]: +def parse_apt_upgradable(output: str) -> list[UpdateDict]: """Parse :data:`APT_UPGRADABLE_COMMAND`'s output, one entry per package. A package listed for several architectures (``libc6`` for amd64 and i386) @@ -75,7 +73,7 @@ def parse_apt_upgradable(output: str) -> List[UpdateDict]: :raises ValueError: when apt failed or its exit status never arrived. """ - by_name: Dict[str, UpdateDict] = {} + by_name: dict[str, UpdateDict] = {} for line in _joined_lines(_listing(output, _APT_STATUS, "apt list --upgradable")): match = _APT_LINE.match(line) if not match: @@ -103,7 +101,7 @@ def nevra_name(nevra: str) -> str: return without_arch.rsplit("-", 2)[0] -def parse_dnf_security(output: str) -> Set[str]: +def parse_dnf_security(output: str) -> set[str]: """The names of the packages a pending security advisory covers. Parses :data:`DNF_SECURITY_COMMAND`'s ``ADVISORY SEVERITY/Sec. NEVRA`` lines. @@ -111,7 +109,7 @@ def parse_dnf_security(output: str) -> Set[str]: :raises ValueError: when dnf failed or its exit status never arrived -- every update would otherwise read as not security. """ - names: Set[str] = set() + names: set[str] = set() for line in _listing(output, _DNF_STATUS, "dnf updateinfo list --security").splitlines(): parts = line.split() if len(parts) >= 3 and parts[1].endswith("/Sec."): diff --git a/napalm_device_types/packages.py b/napalm_device_types/packages.py index 4a8f552..f7c682a 100644 --- a/napalm_device_types/packages.py +++ b/napalm_device_types/packages.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Listing and managing installed software. Identical on every device type that has a package manager -- this was five @@ -12,7 +11,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import PackageDict @@ -20,7 +19,7 @@ from napalm_device_types.models import PackageDict class PackageManagementMixin: if TYPE_CHECKING: - def get_packages(self) -> List[PackageDict]: + def get_packages(self) -> list[PackageDict]: """ Returns all packages currently known to the device's package manager (e.g. ``opkg`` on OpenWrt, ``apk`` on Alpine-based APs). @@ -97,7 +96,7 @@ class PackageManagementMixin: """ ... - def get_package_config(self, name: str) -> Dict[str, Any]: + def get_package_config(self, name: str) -> dict[str, Any]: """ Returns the current configuration of an installed package as a dictionary. The structure is package-specific. @@ -123,7 +122,7 @@ class PackageManagementMixin: """ ... - def set_package_config(self, name: str, config: Dict[str, Any]) -> None: + def set_package_config(self, name: str, config: dict[str, Any]) -> None: """ Writes a new configuration for an installed package. diff --git a/napalm_device_types/phone.py b/napalm_device_types/phone.py index 871a7cd..5409a60 100644 --- a/napalm_device_types/phone.py +++ b/napalm_device_types/phone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for IP telephony endpoints. @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver @@ -36,7 +35,7 @@ class PhoneDriver(DeviceTypeDriver): if TYPE_CHECKING: - def get_sip_accounts(self) -> List[Dict[str, Any]]: + def get_sip_accounts(self) -> list[dict[str, Any]]: """ Returns the SIP registrations configured on the phone. @@ -60,7 +59,7 @@ class PhoneDriver(DeviceTypeDriver): """ ... - def get_call_status(self) -> Dict[str, Any]: + def get_call_status(self) -> dict[str, Any]: """ Returns what the phone is doing right now. diff --git a/napalm_device_types/ping_sweep.py b/napalm_device_types/ping_sweep.py index ddd92c0..1caa9b2 100644 --- a/napalm_device_types/ping_sweep.py +++ b/napalm_device_types/ping_sweep.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Generic ICMP sweep on top of the NAPALM-standard ``ping()``. Sweeping a range of addresses is orchestration, not device mechanics: the @@ -21,7 +20,8 @@ target list (see ``PING_SWEEP_MAX_TARGETS``). from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterable, List, Optional +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any, ClassVar from napalm.base import NetworkDriver @@ -60,7 +60,7 @@ class PingSweepMixin: #: ``False`` opts a driver out of ping sweeps even though it implements #: ``ping()``. ``None`` (the default) means "decide by introspection". - SUPPORTS_PING: ClassVar[Optional[bool]] = None + SUPPORTS_PING: ClassVar[bool | None] = None if TYPE_CHECKING: # pragma: no cover - declared for type checkers only @@ -73,7 +73,7 @@ class PingSweepMixin: size: int = 100, count: int = 5, vrf: str = "", - ) -> Dict[str, Any]: ... + ) -> dict[str, Any]: ... @classmethod def supports_ping(cls) -> bool: @@ -86,9 +86,9 @@ class PingSweepMixin: *, count: int = 1, timeout: int = 1, - max_targets: Optional[int] = None, - on_progress: Optional[Callable[[int, int], None]] = None, - should_stop: Optional[Callable[[], bool]] = None, + max_targets: int | None = None, + on_progress: Callable[[int, int], None] | None = None, + should_stop: Callable[[], bool] | None = None, ) -> PingSweepResultDict: """Ping every address in *destinations* and report who answered. @@ -115,7 +115,7 @@ class PingSweepMixin: targets = targets[:limit] total = len(targets) - entries: List[PingSweepEntryDict] = [] + entries: list[PingSweepEntryDict] = [] for done, destination in enumerate(targets, start=1): if should_stop is not None and should_stop(): break @@ -151,7 +151,12 @@ class PingSweepMixin: success = reply.get("success") or {} if not isinstance(success, dict): - return {"ip": destination, "alive": False, "rtt_ms": None, "error": "malformed ping reply"} + return { + "ip": destination, + "alive": False, + "rtt_ms": None, + "error": "malformed ping reply", + } probes_sent = _as_int(success.get("probes_sent")) packet_loss = _as_int(success.get("packet_loss"), default=probes_sent) alive = bool(success.get("results")) or probes_sent > packet_loss @@ -167,7 +172,7 @@ def _as_int(value: Any, default: int = 0) -> int: return default -def _as_float(value: Any) -> Optional[float]: +def _as_float(value: Any) -> float | None: try: return float(value) except (TypeError, ValueError): diff --git a/napalm_device_types/residential_gateway.py b/napalm_device_types/residential_gateway.py index 369bb7a..b2c9567 100644 --- a/napalm_device_types/residential_gateway.py +++ b/napalm_device_types/residential_gateway.py @@ -19,11 +19,11 @@ Usage:: ... """ -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.dhcp import DhcpServerMixin +from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( HostDict, RadioStatusDict, @@ -31,6 +31,7 @@ from napalm_device_types.models import ( WANStatusDict, WirelessClientDict, ) +from napalm_device_types.nat_vpn import NatVpnMixin class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, DeviceTypeDriver): @@ -48,8 +49,6 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, ROLE: str = "residential_gateway" TYPE_LABEL: str = "Gateway" - - if TYPE_CHECKING: def get_wan_status(self) -> WANStatusDict: @@ -85,7 +84,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_hosts(self) -> List[HostDict]: + def get_hosts(self) -> list[HostDict]: """ Returns the list of hosts known to the gateway (LAN clients). @@ -113,9 +112,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - - - def get_wireless_clients(self) -> List[WirelessClientDict]: + def get_wireless_clients(self) -> list[WirelessClientDict]: """ Returns the list of wireless clients currently associated with the device's built-in access point(s). @@ -125,7 +122,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_ssids(self) -> Dict[str, SSIDDict]: + def get_ssids(self) -> dict[str, SSIDDict]: """ Returns the configured wireless networks (SSIDs). @@ -134,7 +131,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_radio_status(self) -> Dict[str, RadioStatusDict]: + def get_radio_status(self) -> dict[str, RadioStatusDict]: """ Returns the status of the device's wireless radios. diff --git a/napalm_device_types/roles.py b/napalm_device_types/roles.py index dcf8815..914a7f6 100644 --- a/napalm_device_types/roles.py +++ b/napalm_device_types/roles.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Which roles a driver class fills, and which of them is primary. A *role* is what a device is: a firewall, a switch, a NAS. Real devices are @@ -22,7 +21,7 @@ when a concrete driver provided it. from __future__ import annotations -from typing import Any, List, Optional, Type +from typing import Any def _is_role_base(cls: type) -> bool: @@ -36,7 +35,7 @@ def _is_role_base(cls: type) -> bool: return isinstance(role, str) and bool(role) -def roles_of(driver_cls: type) -> List[Type[Any]]: +def roles_of(driver_cls: type) -> list[type[Any]]: """Every role base in ``driver_cls``'s MRO, most significant first. MRO order is inheritance order, so the list reflects exactly what the driver @@ -49,12 +48,12 @@ def roles_of(driver_cls: type) -> List[Type[Any]]: return [cls for cls in roles if not any(o is not cls and issubclass(o, cls) for o in roles)] -def role_keys_of(driver_cls: type) -> List[str]: +def role_keys_of(driver_cls: type) -> list[str]: """The stable string keys of :func:`roles_of`, e.g. ``["storage", "linux"]``.""" return [vars(cls)["ROLE"] for cls in roles_of(driver_cls)] -def primary_role_of(driver_cls: type) -> Optional[str]: +def primary_role_of(driver_cls: type) -> str | None: """The driver's headline role, or ``None`` when it declares no role at all. This is the value netOrk exposes as ``device_class``. diff --git a/napalm_device_types/service_clusters.py b/napalm_device_types/service_clusters.py index fc22784..c117ad7 100644 --- a/napalm_device_types/service_clusters.py +++ b/napalm_device_types/service_clusters.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Service clusters: the configuration that says which services run together. Two hosts that answer at one virtual IP may run the same service by accident. @@ -47,8 +46,9 @@ from __future__ import annotations import ipaddress import json import re +from collections.abc import Callable from shlex import quote -from typing import Callable, Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from urllib.parse import urlsplit from napalm_device_types.models import ( @@ -119,7 +119,7 @@ _PATRONI_SH = ( 'if [ -n "$r" ]; then ' 'b=$(curl -s -k -m 3 "$s://$r/cluster" 2>/dev/null || ' 'wget -q -T 3 --no-check-certificate -O - "$s://$r/cluster" 2>/dev/null); ' - "if [ -n \"$b\" ]; then echo \"rest $(echo \"$b\" | tr -d '\\n')\"; " + 'if [ -n "$b" ]; then echo "rest $(echo "$b" | tr -d \'\\n\')"; ' "else echo rest-unreachable; fi; fi; " 'elif [ -e "$f" ]; then echo "unreadable $f"; ' 'else echo "missing $f"; fi; ' @@ -128,7 +128,7 @@ _PATRONI_SH = ( ) -def _docker_sh(binary: Optional[str]) -> str: +def _docker_sh(binary: str | None) -> str: """The containers part: running containers of the clustered images, their allowlisted environment. ``[no-docker]`` for a host without the engine.""" if not binary: @@ -138,17 +138,17 @@ def _docker_sh(binary: Optional[str]) -> str: 'if command -v "$D" >/dev/null 2>&1; then ' "if c=$(\"$D\" ps --format '{{.Names}}' 2>/dev/null); then echo '[docker]'; " "for n in $c; do " - "i=$(\"$D\" inspect --format '{{.Config.Image}}' \"$n\" 2>/dev/null); " + 'i=$("$D" inspect --format \'{{.Config.Image}}\' "$n" 2>/dev/null); ' 'case "$i" in *adguardhome-sync*|*bunkerweb*) ' 'echo "[container $n]"; echo "image $i"; ' - "\"$D\" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' \"$n\" " + '"$D" inspect --format \'{{range .Config.Env}}{{println .}}{{end}}\' "$n" ' f"2>/dev/null | awk {quote(_ENV_AWK)};; esac; done; " "else echo '[docker-unreadable]'; fi; " "else echo '[no-docker]'; fi; " ) -def service_clusters_command(docker: Optional[str] = "docker") -> str: +def service_clusters_command(docker: str | None = "docker") -> str: """One line of POSIX ``sh``, read-only. *docker* is the engine's binary as the driver reaches it (QNAP's lives elsewhere), None for a host without one. The frame markers are printed in two halves so that a transport which echoes the @@ -165,11 +165,11 @@ def service_clusters_command(docker: Optional[str] = "docker") -> str: SERVICE_CLUSTERS_COMMAND = service_clusters_command() _COMMENT_RE = re.compile(r"\s+#.*$") -_Record = Tuple[str, str] -_Section = Tuple[str, str, List[_Record]] +_Record = tuple[str, str] +_Section = tuple[str, str, list[_Record]] -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -179,9 +179,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> List[_Section]: +def _sections(lines: list[str]) -> list[_Section]: """``[ ]`` headers, each with its records as ``(kind, rest)``.""" - sections: List[_Section] = [] + sections: list[_Section] = [] for line in lines: if line.startswith("[") and line.endswith("]"): kind, _, name = line[1:-1].partition(" ") @@ -202,13 +202,13 @@ def _clean(value: str) -> str: def _member( *, - address: Optional[str] = None, - port: Optional[int] = None, - name: Optional[str] = None, - role: Optional[str] = None, - state: Optional[str] = None, + address: str | None = None, + port: int | None = None, + name: str | None = None, + role: str | None = None, + state: str | None = None, this_host: bool = False, - config_from: Optional[int] = None, + config_from: int | None = None, ) -> ServiceClusterMemberDict: return { "name": name, @@ -225,9 +225,9 @@ def _cluster( mechanism: str, service: str, source: str, - cluster: Optional[str], - members: List[ServiceClusterMemberDict], - running: Optional[bool] = True, + cluster: str | None, + members: list[ServiceClusterMemberDict], + running: bool | None = True, ) -> ServiceClusterDict: return { "mechanism": mechanism, @@ -239,7 +239,7 @@ def _cluster( } -def _host_port(text: str) -> Tuple[Optional[str], Optional[int]]: +def _host_port(text: str) -> tuple[str | None, int | None]: """``"10.0.0.5:8008"`` or ``"[fd00::5]:8008"`` -> host and port.""" try: split = urlsplit(f"//{text.strip()}") @@ -248,7 +248,7 @@ def _host_port(text: str) -> Tuple[Optional[str], Optional[int]]: return None, None -def _url(text: str) -> Tuple[Optional[str], Optional[int]]: +def _url(text: str) -> tuple[str | None, int | None]: """A URL's host and port, the scheme's default where it names none.""" value = text.strip() if "://" not in value: @@ -264,7 +264,7 @@ def _url(text: str) -> Tuple[Optional[str], Optional[int]]: # --- Patroni ------------------------------------------------------------------ -def _patroni_members(rest: str, own: Optional[str]) -> Optional[List[ServiceClusterMemberDict]]: +def _patroni_members(rest: str, own: str | None) -> list[ServiceClusterMemberDict] | None: try: members = json.loads(rest).get("members") except (ValueError, AttributeError): @@ -289,9 +289,9 @@ def _patroni_members(rest: str, own: Optional[str]) -> Optional[List[ServiceClus return sorted(found, key=lambda m: m["name"] or "") -def _patroni(unit: str, records: List[_Record]) -> List[ServiceClusterDict]: - values: Dict[str, str] = {} - rest: Optional[str] = None +def _patroni(unit: str, records: list[_Record]) -> list[ServiceClusterDict]: + values: dict[str, str] = {} + rest: str | None = None for kind, text in records: if kind in ("active", "scope", "name", "restapi") and kind not in values: values[kind] = _clean(text) @@ -313,8 +313,8 @@ def _patroni(unit: str, records: List[_Record]) -> List[ServiceClusterDict]: # --- Containers ---------------------------------------------------------------- -def _env(records: List[_Record]) -> Dict[str, str]: - env: Dict[str, str] = {} +def _env(records: list[_Record]) -> dict[str, str]: + env: dict[str, str] = {} for kind, text in records: if kind == "env": key, _, value = text.partition(" ") @@ -322,13 +322,13 @@ def _env(records: List[_Record]) -> Dict[str, str]: return env -def _image_name(records: List[_Record]) -> str: +def _image_name(records: list[_Record]) -> str: """``bunkerity/bunkerweb@sha256:...`` -> ``bunkerweb``.""" image = next((text for kind, text in records if kind == "image"), "") return image.split("@")[0].rsplit("/", 1)[-1].split(":")[0] -def _adguardhome_sync(name: str, records: List[_Record]) -> Optional[List[ServiceClusterDict]]: +def _adguardhome_sync(name: str, records: list[_Record]) -> list[ServiceClusterDict] | None: env = _env(records) origin = _url(env["ORIGIN_URL"]) if env.get("ORIGIN_URL") else (None, None) if origin[0] is None: @@ -346,7 +346,7 @@ def _adguardhome_sync(name: str, records: List[_Record]) -> Optional[List[Servic return [_cluster("adguardhome-sync", "adguardhome", name, cluster, members)] -def _peer_address(entry: str) -> Optional[str]: +def _peer_address(entry: str) -> str | None: """An entry of a whitelist that names one peer: a single address that is no loopback, link-local or unspecified one -- not a network.""" text = entry.strip() @@ -367,12 +367,12 @@ def _peer_address(entry: str) -> Optional[str]: return str(address) -def _api_port(env: Dict[str, str]) -> int: +def _api_port(env: dict[str, str]) -> int: port = env.get("API_HTTP_PORT", "") return int(port) if port.isdigit() else _BUNKERWEB_API_PORT -def _bunkerweb_worker(name: str, env: Dict[str, str]) -> List[ServiceClusterDict]: +def _bunkerweb_worker(name: str, env: dict[str, str]) -> list[ServiceClusterDict]: managers = [ address for entry in re.split(r"[\s,]+", env.get("API_WHITELIST_IP", "")) @@ -385,7 +385,7 @@ def _bunkerweb_worker(name: str, env: Dict[str, str]) -> List[ServiceClusterDict return [_cluster("bunkerweb", "bunkerweb", name, managers[0], members)] -def _bunkerweb_manager(name: str, env: Dict[str, str]) -> List[ServiceClusterDict]: +def _bunkerweb_manager(name: str, env: dict[str, str]) -> list[ServiceClusterDict]: instances = [entry for entry in env.get("BUNKERWEB_INSTANCES", "").split() if entry] if not instances: return [] @@ -394,12 +394,14 @@ def _bunkerweb_manager(name: str, env: Dict[str, str]) -> List[ServiceClusterDic host, port = _host_port(entry) if host: members.append( - _member(address=host, port=port or _BUNKERWEB_API_PORT, role="worker", config_from=0) + _member( + address=host, port=port or _BUNKERWEB_API_PORT, role="worker", config_from=0 + ) ) return [_cluster("bunkerweb", "bunkerweb", name, None, members)] if len(members) > 1 else [] -def _bunkerweb(name: str, records: List[_Record]) -> List[ServiceClusterDict]: +def _bunkerweb(name: str, records: list[_Record]) -> list[ServiceClusterDict]: image = _image_name(records) if image == "bunkerweb": return _bunkerweb_worker(name, _env(records)) @@ -408,13 +410,13 @@ def _bunkerweb(name: str, records: List[_Record]) -> List[ServiceClusterDict]: return [] # the UI, the API, autoconf: no instance and no manager -_CONTAINER_READERS: Dict[str, Callable[[str, List[_Record]], Optional[List[ServiceClusterDict]]]] = { +_CONTAINER_READERS: dict[str, Callable[[str, list[_Record]], list[ServiceClusterDict] | None]] = { "adguardhome_sync": _adguardhome_sync, "bunkerweb": _bunkerweb, } -def _reader_for(records: List[_Record]) -> Optional[str]: +def _reader_for(records: list[_Record]) -> str | None: image = next((text for kind, text in records if kind == "image"), "") if "adguardhome-sync" in image: return "adguardhome_sync" @@ -423,13 +425,13 @@ def _reader_for(records: List[_Record]) -> Optional[str]: return None -def _containers(sections: List[_Section]) -> Dict[str, Optional[List[ServiceClusterDict]]]: +def _containers(sections: list[_Section]) -> dict[str, list[ServiceClusterDict] | None]: kinds = {kind for kind, _, _ in sections} if "no-docker" in kinds: return {"adguardhome_sync": [], "bunkerweb": []} if "docker" not in kinds: return {"adguardhome_sync": None, "bunkerweb": None} - found: Dict[str, Optional[List[ServiceClusterDict]]] = {"adguardhome_sync": [], "bunkerweb": []} + found: dict[str, list[ServiceClusterDict] | None] = {"adguardhome_sync": [], "bunkerweb": []} for kind, name, records in sections: key = _reader_for(records) if kind == "container" else None if key is None or found[key] is None: @@ -448,14 +450,20 @@ def parse_service_clusters(output: str) -> ServiceClustersDict: :raises ValueError: when the output carries no intact report. """ sections = _sections(_frame(output)) - patroni: Optional[List[ServiceClusterDict]] + patroni: list[ServiceClusterDict] | None if any(kind == "no-systemd" for kind, _, _ in sections): patroni = None - elif any(kind == "patroni" and any(r[0] == "unreadable" for r in records) for kind, _, records in sections): + elif any( + kind == "patroni" and any(r[0] == "unreadable" for r in records) + for kind, _, records in sections + ): patroni = None else: patroni = [ - c for kind, unit, records in sections if kind == "patroni" for c in _patroni(unit, records) + c + for kind, unit, records in sections + if kind == "patroni" + for c in _patroni(unit, records) ] containers = _containers(sections) return { @@ -522,7 +530,9 @@ class ServiceClustersMixin: """ command = f"sh -c {quote(service_clusters_command(self._service_clusters_docker()))}" try: - return parse_service_clusters(self._run_service_clusters_command(command, privileged=True)) + return parse_service_clusters( + self._run_service_clusters_command(command, privileged=True) + ) except ValueError: pass return parse_service_clusters(self._run_service_clusters_command(command, privileged=False)) diff --git a/napalm_device_types/service_packages.py b/napalm_device_types/service_packages.py index f65068a..cf438f1 100644 --- a/napalm_device_types/service_packages.py +++ b/napalm_device_types/service_packages.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Which installed packages a systemd service is made of. A vulnerability is found on a *package*; a statement about a *service* -- the @@ -26,7 +25,7 @@ nothing. from __future__ import annotations import re -from typing import Dict, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING _BEGIN = "SVCPKG_BEGIN" _END = "SVCPKG_END" @@ -61,7 +60,7 @@ SERVICE_PACKAGES_COMMAND = ( ) -def _report(output: str) -> Dict[str, List[str]]: +def _report(output: str) -> dict[str, list[str]]: lines = [line.rstrip() for line in output.splitlines()] stripped = [line.strip() for line in lines] try: @@ -69,8 +68,8 @@ def _report(output: str) -> Dict[str, List[str]]: end = stripped.index(_END, start) except ValueError: raise ValueError("no intact service package report in the output") from None - sections: Dict[str, List[str]] = {} - current: Optional[List[str]] = None + sections: dict[str, list[str]] = {} + current: list[str] | None = None for line in lines[start + 1 : end]: if line in ("[units]", "[dpkg]", "[rpm]"): current = sections.setdefault(line[1:-1], []) @@ -79,11 +78,11 @@ def _report(output: str) -> Dict[str, List[str]]: return sections -def _units(lines: List[str]) -> Dict[str, Set[str]]: +def _units(lines: list[str]) -> dict[str, set[str]]: """``systemctl show`` blocks -> ``{unit: its unit file and binary}``.""" - units: Dict[str, Set[str]] = {} + units: dict[str, set[str]] = {} unit = "" - paths: Set[str] = set() + paths: set[str] = set() for line in [*lines, ""]: key, _, value = line.partition("=") if not line.strip(): @@ -101,7 +100,7 @@ def _units(lines: List[str]) -> Dict[str, Set[str]]: return units -def _names(raw: str) -> Set[str]: +def _names(raw: str) -> set[str]: """``libfoo1:amd64, foo-bin`` -> ``{"libfoo1", "foo-bin"}``; nothing else counts.""" found = set() for name in raw.replace(",", " ").split(): @@ -111,9 +110,9 @@ def _names(raw: str) -> Set[str]: return found -def _owners(sections: Dict[str, List[str]]) -> Dict[str, Set[str]]: +def _owners(sections: dict[str, list[str]]) -> dict[str, set[str]]: """``{path: the packages owning it}`` from dpkg's or rpm's answer.""" - owners: Dict[str, Set[str]] = {} + owners: dict[str, set[str]] = {} for line in sections.get("dpkg", []): # "pkg[, pkg2][:arch]: /path"; "diversion by x from: /path" is no owner. if line.startswith("diversion by") or ": /" not in line: @@ -127,11 +126,11 @@ def _owners(sections: Dict[str, List[str]]) -> Dict[str, Set[str]]: return owners -def _across_usr_merge(path: str) -> Tuple[str, str]: +def _across_usr_merge(path: str) -> tuple[str, str]: return (path, path[4:] if path.startswith("/usr/") else "/usr" + path) -def parse_service_packages(output: str) -> Dict[str, List[str]]: +def parse_service_packages(output: str) -> dict[str, list[str]]: """Parse what :data:`SERVICE_PACKAGES_COMMAND` printed. ``{unit name without ".service": its packages, sorted}``. A unit no package @@ -163,7 +162,7 @@ class ServicePackagesMixin: """Run *command* on the host with ``sh`` and return what it printed.""" ... - def get_service_packages(self) -> Dict[str, List[str]]: + def get_service_packages(self) -> dict[str, list[str]]: """ Returns, per systemd service, the installed packages it is made of. diff --git a/napalm_device_types/services.py b/napalm_device_types/services.py index 1c8a490..5e43ec2 100644 --- a/napalm_device_types/services.py +++ b/napalm_device_types/services.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Init-system services: listing them and starting/stopping them. Deliberately *not* the same thing as a NAS's exported shares, which live on @@ -13,7 +12,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ServiceDict @@ -21,7 +20,7 @@ from napalm_device_types.models import ServiceDict class ServiceControlMixin: if TYPE_CHECKING: - def get_services(self) -> List[ServiceDict]: + def get_services(self) -> list[ServiceDict]: """ Returns the list of system services known to the device's init system. @@ -42,7 +41,7 @@ class ServiceControlMixin: """ ... - def manage_service(self, name: str, action: str) -> Dict[str, Any]: + def manage_service(self, name: str, action: str) -> dict[str, Any]: """ Execute a lifecycle action on a named service. diff --git a/napalm_device_types/ssh_host_keys.py b/napalm_device_types/ssh_host_keys.py index 5c86e3f..c1ee214 100644 --- a/napalm_device_types/ssh_host_keys.py +++ b/napalm_device_types/ssh_host_keys.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """SSH host keys: trust on first use, and a changed key refused before any login. A driver that accepts whatever host key answers at a device's address logs in @@ -41,9 +40,10 @@ import hashlib import logging import os import tempfile +from collections.abc import Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass -from typing import Any, Dict, Iterator, List, Mapping, Optional, Set +from typing import Any from napalm.base.exceptions import ConnectionException @@ -74,11 +74,11 @@ class SshHostKey: digest = hashlib.sha256(base64.b64decode(self.key)).digest() return "SHA256:" + base64.b64encode(digest).decode("ascii").rstrip("=") - def as_dict(self) -> Dict[str, str]: + def as_dict(self) -> dict[str, str]: return {"key_type": self.key_type, "key": self.key, "fingerprint": self.fingerprint} @classmethod - def from_value(cls, value: Any) -> Optional["SshHostKey"]: + def from_value(cls, value: Any) -> SshHostKey | None: """The key in *value* (a mapping with ``key_type`` and ``key``), or None for none.""" if not value: return None @@ -87,7 +87,7 @@ class SshHostKey: return cls(str(value["key_type"]), str(value["key"])) @classmethod - def of(cls, pkey: Any) -> "SshHostKey": + def of(cls, pkey: Any) -> SshHostKey: """The key of a paramiko ``PKey`` (what a transport says the server presented).""" return cls(pkey.get_name(), pkey.get_base64()) @@ -104,14 +104,14 @@ class HostKeyObservation: host: str port: int seen: SshHostKey - expected: Optional[SshHostKey] + expected: SshHostKey | None @property def accepted(self) -> bool: """True on first use and for the expected key; False for a refused one.""" return self.expected is None or self.expected == self.seen - def as_dict(self) -> Dict[str, Any]: + def as_dict(self) -> dict[str, Any]: return { "host": self.host, "port": self.port, @@ -161,7 +161,7 @@ class _RefuseUnknown: raise ConnectionException(f"SSH host key of {hostname} is not the one expected") -def _expected_key(known: Any, port: int) -> Optional[SshHostKey]: +def _expected_key(known: Any, port: int) -> SshHostKey | None: """The key in *known* (``{port: key}``, the port as text or number) for *port*.""" if not known: return None @@ -170,10 +170,10 @@ def _expected_key(known: Any, port: int) -> Optional[SshHostKey]: return SshHostKey.from_value(known.get(str(port)) or known.get(port)) -def _refused_key(exc: BaseException) -> Optional[Any]: +def _refused_key(exc: BaseException) -> Any | None: """paramiko's ``BadHostKeyException`` in *exc*'s chain (netmiko and drivers wrap it).""" - seen: Set[int] = set() - stack: List[Optional[BaseException]] = [exc] + seen: set[int] = set() + stack: list[BaseException | None] = [exc] while stack: current = stack.pop() if current is None or id(current) in seen: @@ -192,7 +192,7 @@ class HostKeyCheck: address the connection goes to, as given to paramiko or netmiko. """ - def __init__(self, optional_args: Optional[Mapping[str, Any]], host: str, port: int) -> None: + def __init__(self, optional_args: Mapping[str, Any] | None, host: str, port: int) -> None: args = optional_args or {} self.host = host self.port = int(port) @@ -228,7 +228,7 @@ class HostKeyCheck: # ------------------------------------------------------------------ @contextmanager - def netmiko(self, base: Mapping[str, Any]) -> Iterator[Dict[str, Any]]: + def netmiko(self, base: Mapping[str, Any]) -> Iterator[dict[str, Any]]: """``ConnectHandler`` keyword arguments: *base* with the host key decided. Open the connection inside the block and call :meth:`observe` on its diff --git a/napalm_device_types/ssh_server.py b/napalm_device_types/ssh_server.py index 16857d7..e8e33f7 100644 --- a/napalm_device_types/ssh_server.py +++ b/napalm_device_types/ssh_server.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """OpenSSH server hardening: taking algorithms out of what sshd offers. A scanner such as ssh-audit names algorithms a server should stop offering -- @@ -36,15 +35,18 @@ Only without such an ``Include`` is sshd_config itself rewritten, after a copy. from __future__ import annotations import time +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field from shlex import quote -from typing import Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, TYPE_CHECKING +from typing import ( + TYPE_CHECKING, +) if TYPE_CHECKING: # pragma: no cover - for type checkers only from napalm_device_types.channel import CommandResult #: The algorithm classes: key -> (``sshd -T`` keyword, sshd_config directive). -SSH_ALGORITHM_CLASSES: Dict[str, Tuple[str, str]] = { +SSH_ALGORITHM_CLASSES: dict[str, tuple[str, str]] = { "kex": ("kexalgorithms", "KexAlgorithms"), "hostkey": ("hostkeyalgorithms", "HostKeyAlgorithms"), "cipher": ("ciphers", "Ciphers"), @@ -65,7 +67,7 @@ _HEADER = ( #: Host key type, as ``ssh-keygen -l`` prints it -> the algorithms that key serves. #: Certificate algorithms are left out: they also need a HostCertificate. -_KEY_TYPE_ALGORITHMS: Dict[str, Tuple[str, ...]] = { +_KEY_TYPE_ALGORITHMS: dict[str, tuple[str, ...]] = { "ED25519": ("ssh-ed25519",), "RSA": ("rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"), "DSA": ("ssh-dss",), @@ -82,7 +84,7 @@ _RELOAD = ( ) -def _directive_key(line: str) -> Optional[str]: +def _directive_key(line: str) -> str | None: """The algorithm class a global sshd_config line sets, if it sets one.""" parts = line.split(None, 1) if not parts: @@ -94,12 +96,12 @@ def _directive_key(line: str) -> Optional[str]: return None -def parse_sshd_effective(text: str) -> Dict[str, List[str]]: +def parse_sshd_effective(text: str) -> dict[str, list[str]]: """The four algorithm lists from ``sshd -T`` output, in the server's order. :raises ValueError: if a list is missing -- not sshd -T output, or cut short. """ - found: Dict[str, List[str]] = {} + found: dict[str, list[str]] = {} for raw in text.splitlines(): parts = raw.strip().split(None, 1) if len(parts) != 2: @@ -113,9 +115,9 @@ def parse_sshd_effective(text: str) -> Dict[str, List[str]]: return found -def parse_sshd_hostkeys(text: str) -> List[str]: +def parse_sshd_hostkeys(text: str) -> list[str]: """The ``hostkey`` paths from ``sshd -T`` output, in order.""" - paths: List[str] = [] + paths: list[str] = [] for raw in text.splitlines(): parts = raw.strip().split(None, 1) if len(parts) == 2 and parts[0].lower() == "hostkey": @@ -123,12 +125,12 @@ def parse_sshd_hostkeys(text: str) -> List[str]: return paths -def host_key_algorithms(fingerprints: str) -> Optional[Set[str]]: +def host_key_algorithms(fingerprints: str) -> set[str] | None: """The host key algorithms the keys in ``ssh-keygen -l`` output can serve. None when a line is not understood or there is no key: then nothing is known. """ - usable: Set[str] = set() + usable: set[str] = set() for raw in fingerprints.splitlines(): parts = raw.split() if not parts: @@ -163,13 +165,13 @@ class Refusal: @dataclass(frozen=True) class SshHardeningPlan: - keep: Dict[str, List[str]] - remove: Dict[str, List[str]] - refused: List[Refusal] = field(default_factory=list) - not_offered: List[Refusal] = field(default_factory=list) + keep: dict[str, list[str]] + remove: dict[str, list[str]] + refused: list[Refusal] = field(default_factory=list) + not_offered: list[Refusal] = field(default_factory=list) @property - def changed_classes(self) -> List[str]: + def changed_classes(self) -> list[str]: return [k for k in SSH_ALGORITHM_CLASSES if self.remove.get(k)] @@ -177,8 +179,8 @@ def plan_ssh_hardening( effective: Mapping[str, Sequence[str]], remove: Mapping[str, Iterable[str]], *, - client: Optional[Mapping[str, Iterable[str]]] = None, - host_keys: Optional[Iterable[str]] = None, + client: Mapping[str, Iterable[str]] | None = None, + host_keys: Iterable[str] | None = None, ) -> SshHardeningPlan: """What the lists become when *remove* is taken out of *effective*. @@ -199,10 +201,10 @@ def plan_ssh_hardening( unknown = [k for k in remove if k not in SSH_ALGORITHM_CLASSES] if unknown: raise ValueError(f"unknown SSH algorithm class: {', '.join(unknown)}") - keep: Dict[str, List[str]] = {} - removed: Dict[str, List[str]] = {} - refused: List[Refusal] = [] - not_offered: List[Refusal] = [] + keep: dict[str, list[str]] = {} + removed: dict[str, list[str]] = {} + refused: list[Refusal] = [] + not_offered: list[Refusal] = [] usable_keys = set(host_keys) if host_keys is not None else None for kind in SSH_ALGORITHM_CLASSES: offered = list(effective.get(kind, [])) @@ -245,7 +247,7 @@ def render_dropin(keep: Mapping[str, Sequence[str]], kinds: Iterable[str]) -> st return _HEADER + "".join(lines) -def dropin_classes(text: str) -> List[str]: +def dropin_classes(text: str) -> list[str]: """The classes an existing drop-in sets.""" return [k for k in (_directive_key(line.strip()) for line in text.splitlines()) if k] @@ -268,11 +270,11 @@ def has_dropin_include(text: str) -> bool: return False -def _split_our_block(text: str) -> Tuple[List[str], str]: +def _split_our_block(text: str) -> tuple[list[str], str]: """(the classes an earlier rewrite put above the file, the file without them).""" if not text.startswith(_HEADER): return [], text - lines = text[len(_HEADER):].splitlines(keepends=True) + lines = text[len(_HEADER) :].splitlines(keepends=True) ours = 0 while ours < len(lines) and _directive_key(lines[ours].strip()): ours += 1 @@ -288,7 +290,7 @@ def render_main_config(text: str, keep: Mapping[str, Sequence[str]], kinds: Iter """ earlier, body = _split_our_block(text) wanted = set(kinds) | set(earlier) - out: List[str] = [] + out: list[str] = [] in_match = False for raw in body.splitlines(): if _is_match(raw.strip()): @@ -312,8 +314,8 @@ class SshHardeningResult: plan: SshHardeningPlan applied: bool = False rolled_back: bool = False - path: Optional[str] = None - backup: Optional[str] = None + path: str | None = None + backup: str | None = None message: str = "" @@ -336,19 +338,19 @@ class OpenSshServerMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: ... # ── the host ──────────────────────────────────────────────────────── - def _ssh_run(self, command: str, stdin: Optional[bytes] = None) -> "CommandResult": + def _ssh_run(self, command: str, stdin: bytes | None = None) -> CommandResult: return self.run_command(command, privileged=True, timeout=30, stdin=stdin) - def _ssh_read(self, path: str) -> Optional[str]: + def _ssh_read(self, path: str) -> str | None: result = self._ssh_run(f"cat -- {quote(path)}") return result.stdout if result.exit_code == 0 else None - def _ssh_write(self, path: str, content: str, mode_of: Optional[str] = None) -> None: + def _ssh_write(self, path: str, content: str, mode_of: str | None = None) -> None: """Replace *path* with *content*, as root, without root reading stdin. A driver gains root with ``sudo -S`` and puts the password in front of @@ -368,17 +370,21 @@ class OpenSshServerMixin: ) tmp = staged.stdout.strip() if staged.exit_code != 0 or not tmp.startswith("/tmp/netork-ssh."): - raise RuntimeError(f"could not stage {path}: {(staged.stderr or staged.stdout).strip()}") + raise RuntimeError( + f"could not stage {path}: {(staged.stderr or staged.stdout).strip()}" + ) part = path + ".netork-tmp" result = self._ssh_run( # GNU stat, then BSD's; 644 for a file that does not exist yet. - f"r={quote(mode_of or path)}; m=$(stat -c %a -- \"$r\" 2>/dev/null" + f'r={quote(mode_of or path)}; m=$(stat -c %a -- "$r" 2>/dev/null' f' || stat -f %Lp -- "$r" 2>/dev/null) || m=644;' f' install -m "$m" -- {quote(tmp)} {quote(part)} && mv -f -- {quote(part)} {quote(path)};' f" s=$?; rm -f -- {quote(tmp)} {quote(part)}; exit $s" ) if result.exit_code != 0: - raise RuntimeError(f"could not write {path}: {(result.stderr or result.stdout).strip()}") + raise RuntimeError( + f"could not write {path}: {(result.stderr or result.stdout).strip()}" + ) def _ssh_remove(self, path: str) -> None: self._ssh_run(f"rm -f -- {quote(path)}") @@ -389,7 +395,7 @@ class OpenSshServerMixin: if result.exit_code != 0: raise RuntimeError(f"could not reload sshd: {(result.stderr or result.stdout).strip()}") - def get_ssh_server_algorithms(self) -> Dict[str, List[str]]: + def get_ssh_server_algorithms(self) -> dict[str, list[str]]: """The algorithm lists sshd uses now (``sshd -T``), per class. :raises RuntimeError: if sshd -T fails -- no OpenSSH, or no root. @@ -402,7 +408,7 @@ class OpenSshServerMixin: raise RuntimeError(f"sshd -T failed: {(result.stderr or result.stdout).strip()}") return result.stdout - def _ssh_host_key_algorithms(self, sshd_t: str) -> Optional[Set[str]]: + def _ssh_host_key_algorithms(self, sshd_t: str) -> set[str] | None: """The host key algorithms the host has a key for; None when that cannot be read. A ``HostKey`` whose file does not exist is skipped, as sshd skips it. @@ -418,7 +424,7 @@ class OpenSshServerMixin: return None return host_key_algorithms(result.stdout) - def _sshd_accepts(self) -> Optional[str]: + def _sshd_accepts(self) -> str | None: """None when ``sshd -t`` accepts the configuration, else what it said.""" result = self._ssh_run("PATH=$PATH:/usr/sbin:/sbin sshd -t") return None if result.exit_code == 0 else (result.stderr or result.stdout).strip() @@ -429,9 +435,9 @@ class OpenSshServerMixin: self, remove: Mapping[str, Iterable[str]], *, - client: Optional[Mapping[str, Iterable[str]]] = None, - verify: Optional[Callable[[], bool]] = None, - log: Optional[Callable[[str], None]] = None, + client: Mapping[str, Iterable[str]] | None = None, + verify: Callable[[], bool] | None = None, + log: Callable[[str], None] | None = None, ) -> SshHardeningResult: """Stop the server offering the algorithms in *remove* (class -> names). @@ -447,11 +453,13 @@ class OpenSshServerMixin: say = log or (lambda _line: None) say("Reading the algorithms sshd offers (sshd -T)") sshd_t = self._sshd_effective_text() - host_keys: Optional[Set[str]] = None + host_keys: set[str] | None = None if remove.get("hostkey"): say("Reading which host keys the server has (ssh-keygen -l)") host_keys = self._ssh_host_key_algorithms(sshd_t) - plan = plan_ssh_hardening(parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys) + plan = plan_ssh_hardening( + parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys + ) whys = { "last": "it is the last one", "client": "netOrk could no longer connect", @@ -477,7 +485,11 @@ class OpenSshServerMixin: return self._apply_or_restore(plan, kinds, result, restore, verify, say) def _write_change( - self, plan: SshHardeningPlan, main: str, result: SshHardeningResult, say: Callable[[str], None] + self, + plan: SshHardeningPlan, + main: str, + result: SshHardeningResult, + say: Callable[[str], None], ) -> Callable[[], None]: """Write the new lists; return what puts the previous state back.""" if has_dropin_include(main): @@ -509,10 +521,10 @@ class OpenSshServerMixin: def _apply_or_restore( self, plan: SshHardeningPlan, - kinds: List[str], + kinds: list[str], result: SshHardeningResult, restore: Callable[[], None], - verify: Optional[Callable[[], bool]], + verify: Callable[[], bool] | None, say: Callable[[str], None], ) -> SshHardeningResult: def back(message: str, reload: bool) -> SshHardeningResult: diff --git a/napalm_device_types/storage.py b/napalm_device_types/storage.py index 6ea52c4..1455221 100644 --- a/napalm_device_types/storage.py +++ b/napalm_device_types/storage.py @@ -10,9 +10,9 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.models import ( DiskPoolDict, LogicalVolumeDict, @@ -23,6 +23,7 @@ from napalm_device_types.models import ( StorageServiceDict, VolumeSnapshotDict, ) +from napalm_device_types.packages import PackageManagementMixin class StorageDriver(PackageManagementMixin, DeviceTypeDriver): @@ -47,7 +48,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): if TYPE_CHECKING: - def get_disks(self) -> List[PhysicalDiskDict]: + def get_disks(self) -> list[PhysicalDiskDict]: """ Returns all physical drives detected by the storage device. @@ -95,7 +96,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): """ ... - def get_disk_pools(self) -> Dict[str, DiskPoolDict]: + def get_disk_pools(self) -> dict[str, DiskPoolDict]: """ Returns the disk pools (RAID arrays, ZFS pools, LVM volume groups, etc.) configured on the device. @@ -140,7 +141,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Logical volumes / datasets # ------------------------------------------------------------------ - def get_volumes(self) -> Dict[str, LogicalVolumeDict]: + def get_volumes(self) -> dict[str, LogicalVolumeDict]: """ Returns all logical volumes, ZFS datasets, or LUNs on the device. @@ -196,7 +197,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Shares # ------------------------------------------------------------------ - def get_shares(self) -> Dict[str, NASShareDict]: + def get_shares(self) -> dict[str, NASShareDict]: """ Returns all network shares and iSCSI targets exported by the device. @@ -244,7 +245,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Snapshots # ------------------------------------------------------------------ - def get_volume_snapshots(self, volume: str = "") -> List[VolumeSnapshotDict]: + def get_volume_snapshots(self, volume: str = "") -> list[VolumeSnapshotDict]: """ Returns volume / dataset snapshots. @@ -339,7 +340,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Quotas # ------------------------------------------------------------------ - def get_quotas(self) -> List[StorageQuotaDict]: + def get_quotas(self) -> list[StorageQuotaDict]: """ Returns all filesystem quotas configured on the device. @@ -380,7 +381,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Services # ------------------------------------------------------------------ - def get_storage_services(self) -> Dict[str, StorageServiceDict]: + def get_storage_services(self) -> dict[str, StorageServiceDict]: """ Returns the file-sharing and access services available on the device. @@ -443,7 +444,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Replication # ------------------------------------------------------------------ - def get_replication_jobs(self) -> List[ReplicationJobDict]: + def get_replication_jobs(self) -> list[ReplicationJobDict]: """ Returns all replication / sync jobs configured on the device. @@ -493,8 +494,3 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # ------------------------------------------------------------------ # Package management (plugins / extensions) # ------------------------------------------------------------------ - - - - - diff --git a/napalm_device_types/switch.py b/napalm_device_types/switch.py index 07eee15..704c4f2 100644 --- a/napalm_device_types/switch.py +++ b/napalm_device_types/switch.py @@ -10,7 +10,9 @@ Usage:: ... """ -from typing import Any, Awaitable, Callable, Dict, List, TYPE_CHECKING +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any + from napalm_device_types.base import DeviceTypeDriver from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.models import ( @@ -45,7 +47,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): async def get_health_metrics( cls, snmp_get: Callable[..., Awaitable[Any]], - snmp_walk: Callable[..., Awaitable[Dict[str, Any]]], + snmp_walk: Callable[..., Awaitable[dict[str, Any]]], ) -> HealthMetricsDict: """Collect SNMP health metrics for this switch type. @@ -54,7 +56,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def get_spanning_tree(self) -> Dict[str, SpanningTreeDict]: + def get_spanning_tree(self) -> dict[str, SpanningTreeDict]: """ Returns spanning tree status for each STP instance. @@ -104,7 +106,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def get_port_channels(self) -> Dict[str, PortChannelDict]: + def get_port_channels(self) -> dict[str, PortChannelDict]: """ Returns port-channel (LAG) configuration and status. @@ -129,8 +131,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - - def get_dot1x_ports(self) -> Dict[str, Dot1XPortDict]: + def get_dot1x_ports(self) -> dict[str, Dot1XPortDict]: """ Returns the 802.1X / NAC configuration per switch port. @@ -350,7 +351,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def set_lag_members(self, lag_name: str, members: List[str]) -> None: + def set_lag_members(self, lag_name: str, members: list[str]) -> None: """ Sets the full member-port list of a LAG/trunk interface. diff --git a/napalm_device_types/system_identity.py b/napalm_device_types/system_identity.py index 045c8a2..6549d3b 100644 --- a/napalm_device_types/system_identity.py +++ b/napalm_device_types/system_identity.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The SMBIOS identity of a host: system UUID, product serial, board serial. What ties a server to its BMC. The BMC reports the server's serial number and @@ -32,19 +31,37 @@ _DMI = "/sys/class/dmi/id" #: output cannot shift a field. SYSTEM_IDENTITY_COMMAND = ( f"printf '{_BEGIN}\\n%s\\n%s\\n%s\\n{_END}\\n' " - f"\"$(cat {_DMI}/product_uuid 2>/dev/null)\" " - f"\"$(cat {_DMI}/product_serial 2>/dev/null)\" " - f"\"$(cat {_DMI}/board_serial 2>/dev/null)\"" + f'"$(cat {_DMI}/product_uuid 2>/dev/null)" ' + f'"$(cat {_DMI}/product_serial 2>/dev/null)" ' + f'"$(cat {_DMI}/board_serial 2>/dev/null)"' ) #: Lowercased defaults board vendors leave in place. -_PLACEHOLDERS = frozenset({ - "", "none", "n/a", "na", "not specified", "not applicable", "not available", - "to be filled by o.e.m.", "to be filled by oem", "default string", "default", - "system serial number", "base board serial number", "chassis serial number", - "serial number", "0123456789", "1234567890", "123456789", "o.e.m.", "oem", - "03000200-0400-0500-0006-000700080009", -}) +_PLACEHOLDERS = frozenset( + { + "", + "none", + "n/a", + "na", + "not specified", + "not applicable", + "not available", + "to be filled by o.e.m.", + "to be filled by oem", + "default string", + "default", + "system serial number", + "base board serial number", + "chassis serial number", + "serial number", + "0123456789", + "1234567890", + "123456789", + "o.e.m.", + "oem", + "03000200-0400-0500-0006-000700080009", + } +) #: A value of one repeated character ("0", "00000000", "FFFF-...") or of zeros and Fs. _REPEATED = re.compile(r"^(.)\1*$") diff --git a/napalm_device_types/systemd.py b/napalm_device_types/systemd.py index d181d4b..1089818 100644 --- a/napalm_device_types/systemd.py +++ b/napalm_device_types/systemd.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """systemd services: listing them in one round trip, and starting and stopping them. What systemd reports about its services, and how one is started or stopped, is @@ -31,7 +30,7 @@ from __future__ import annotations import re from shlex import quote -from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ServiceDict from napalm_device_types.services import ServiceControlMixin @@ -131,7 +130,7 @@ def service_action_command(name: str, action: str) -> str: ) -def parse_action_result(output: str) -> Dict[str, Any]: +def parse_action_result(output: str) -> dict[str, Any]: """``{"success", "output"}`` from what :func:`service_action_command` printed. Only the exit status decides. A job still running when ``timeout`` gave up @@ -151,7 +150,7 @@ def parse_action_result(output: str) -> Dict[str, Any]: return {"success": False, "output": text or f"systemctl exited with status {status}."} -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -161,9 +160,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: List[str] = [] +def _sections(lines: list[str]) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines: if line.startswith("[") and line.endswith("]"): current = sections.setdefault(line[1:-1], []) @@ -172,15 +171,15 @@ def _sections(lines: List[str]) -> Dict[str, List[str]]: return sections -def _unit_blocks(lines: List[str]) -> List[Dict[str, str]]: +def _unit_blocks(lines: list[str]) -> list[dict[str, str]]: """``systemctl show``'s output, one dict per unit. Units are separated by a blank line -- except where ``xargs`` split the list over two runs and the blocks meet, so a key seen twice starts the next unit as well. """ - blocks: List[Dict[str, str]] = [] - current: Dict[str, str] = {} + blocks: list[dict[str, str]] = [] + current: dict[str, str] = {} for line in lines: key, sep, value = line.partition("=") if not sep or key in current: @@ -198,17 +197,17 @@ def _base(unit: str) -> str: return unit[: -len(_SUFFIX)] -def _main_pid(block: Dict[str, str]) -> int: +def _main_pid(block: dict[str, str]) -> int: try: return int(block.get("MainPID") or 0) except ValueError: return 0 -def _loaded(blocks: List[Dict[str, str]]) -> Tuple[Dict[str, ServiceDict], Set[str]]: +def _loaded(blocks: list[dict[str, str]]) -> tuple[dict[str, ServiceDict], set[str]]: """The loaded services, and every name they go by (aliases included).""" - services: Dict[str, ServiceDict] = {} - names: Set[str] = set() + services: dict[str, ServiceDict] = {} + names: set[str] = set() for block in blocks: unit = block.get("Id", "") if not unit.endswith(_SUFFIX) or block.get("LoadState") == "not-found": @@ -224,14 +223,14 @@ def _loaded(blocks: List[Dict[str, str]]) -> Tuple[Dict[str, ServiceDict], Set[s return services, names -def _installed(lines: List[str], known: Set[str]) -> Dict[str, ServiceDict]: +def _installed(lines: list[str], known: set[str]) -> dict[str, ServiceDict]: """Installed services that are not loaded: neither running nor starting now. Templates, static units and aliases are left out -- the last also when an older systemd lists an alias as ``enabled``, which is why every name a loaded unit goes by is skipped. """ - services: Dict[str, ServiceDict] = {} + services: dict[str, ServiceDict] = {} for line in lines: parts = line.split() if len(parts) < 2: @@ -253,7 +252,7 @@ def _installed(lines: List[str], known: Set[str]) -> Dict[str, ServiceDict]: return services -def _apply_generated(services: Dict[str, ServiceDict], lines: List[str]) -> None: +def _apply_generated(services: dict[str, ServiceDict], lines: list[str]) -> None: """Take a generated unit's boot state from ``is-enabled``'s answer.""" for line in lines: parts = line.split() @@ -261,7 +260,7 @@ def _apply_generated(services: Dict[str, ServiceDict], lines: List[str]) -> None services[_base(parts[0])]["enabled"] = parts[1] in _ENABLED -def parse_systemd_services(output: str) -> List[ServiceDict]: +def parse_systemd_services(output: str) -> list[ServiceDict]: """Parse what :data:`SYSTEMD_SERVICES_COMMAND` printed, sorted by name. Lists every loaded service unit but those that are not found, and every @@ -299,7 +298,7 @@ class SystemdServicesMixin(ServiceControlMixin): """ ... - def get_services(self) -> List[ServiceDict]: + def get_services(self) -> list[ServiceDict]: """ Returns the services systemd knows, in one round trip. @@ -316,7 +315,7 @@ class SystemdServicesMixin(ServiceControlMixin): ) return parse_systemd_services(output) - def manage_service(self, name: str, action: str) -> Dict[str, Any]: + def manage_service(self, name: str, action: str) -> dict[str, Any]: """ Applies *action* (start, stop, restart, enable, disable) to the service *name*. diff --git a/napalm_device_types/terminal.py b/napalm_device_types/terminal.py index c5e4c00..67aa284 100644 --- a/napalm_device_types/terminal.py +++ b/napalm_device_types/terminal.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """What a pseudo-terminal adds to a command's output, taken out again. A screen-scraping transport (netmiko) gives the remote command a terminal. Tools diff --git a/napalm_device_types/updates.py b/napalm_device_types/updates.py index 90059b4..7660181 100644 --- a/napalm_device_types/updates.py +++ b/napalm_device_types/updates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Pending package updates and applying them. The two device types that offer this used to declare it under two different @@ -13,7 +12,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict, UpdateProgressDict @@ -21,7 +20,7 @@ from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict, Updat class UpdateMixin: if TYPE_CHECKING: - def get_available_updates(self) -> List[UpdateDict]: + def get_available_updates(self) -> list[UpdateDict]: """ Returns a list of packages that have a newer version available. @@ -51,7 +50,7 @@ class UpdateMixin: """ ... - def refresh_available_updates(self) -> Dict[str, Any]: + def refresh_available_updates(self) -> dict[str, Any]: """ Refreshes the host's package index, so that :meth:`get_available_updates` reports what the repositories offer now (``apt-get update``, @@ -73,7 +72,7 @@ class UpdateMixin: """ ... - def apply_updates(self, packages: List[str]) -> ApplyUpdatesResultDict: + def apply_updates(self, packages: list[str]) -> ApplyUpdatesResultDict: """ Upgrades the given packages to the newest available version. diff --git a/napalm_device_types/virtual_ips.py b/napalm_device_types/virtual_ips.py index 6733684..a8bb46b 100644 --- a/napalm_device_types/virtual_ips.py +++ b/napalm_device_types/virtual_ips.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Virtual IPs: the addresses a host's HA configuration lets float between machines. vip-manager and keepalived put a service address on whichever member is master @@ -51,9 +50,10 @@ from __future__ import annotations import ipaddress import re +from collections.abc import Callable from dataclasses import dataclass, field from shlex import quote -from typing import Callable, Dict, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import VirtualIpDict, VirtualIpsDict from napalm_device_types.terminal import strip_terminal_codes @@ -108,7 +108,7 @@ _AWK = " ".join( r" if (!ok(f)) return; seen[f] = 1;" r' while (0 < (r = (getline l < f))) { sub(/[#!].*/, "", l); gsub(/[{}]/, " & ", l);' r' n = split(l, t, " "); if (n && nw && t[1] != "{") { stmt(w, nw, f, d); nw = 0 }' - r' for (k = 1; k <= n; k++) {' + r" for (k = 1; k <= n; k++) {" r' if (t[k] == "{") { push(nw ? w[1] : "", 1 < nw ? w[2] : ""); nw = 0 }' r' else if (t[k] == "}") { stmt(w, nw, f, d); nw = 0; pop() } else w[++nw] = t[k] } }' r" stmt(w, nw, f, d);" @@ -157,10 +157,10 @@ VIRTUAL_IPS_COMMAND = ( _COMMENT_RE = re.compile(r"\s+#.*$") _VARIABLE_RE = re.compile(r"^\$\{?(\w+)\}?$") -_Record = Tuple[str, str] +_Record = tuple[str, str] -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -170,9 +170,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> List[Tuple[str, str, List[_Record]]]: +def _sections(lines: list[str]) -> list[tuple[str, str, list[_Record]]]: """``[ ]`` headers, each with its records as ``(kind, rest)``.""" - sections: List[Tuple[str, str, List[_Record]]] = [] + sections: list[tuple[str, str, list[_Record]]] = [] for line in lines: if line.startswith("[") and line.endswith("]"): mechanism, _, unit = line[1:-1].partition(" ") @@ -191,7 +191,7 @@ def _clean(value: str) -> str: return value.strip() -def _address(text: str) -> Optional[Tuple[str, Optional[int]]]: +def _address(text: str) -> tuple[str, int | None] | None: """``"10.0.0.10/24"`` -> ``("10.0.0.10", 24)``; None for what is no address.""" host, slash, prefix = text.partition("/") try: @@ -205,7 +205,7 @@ def _address(text: str) -> Optional[Tuple[str, Optional[int]]]: return str(address), int(prefix) -def _prefix(mask: Optional[str]) -> Optional[int]: +def _prefix(mask: str | None) -> int | None: """``"24"`` or ``"255.255.255.0"`` -> 24.""" if not mask: return None @@ -217,27 +217,27 @@ def _prefix(mask: Optional[str]) -> Optional[int]: return None -def _running(records: List[_Record]) -> Optional[bool]: +def _running(records: list[_Record]) -> bool | None: states = [rest for kind, rest in records if kind == "active"] return states[-1] == "active" if states else None -def _vip_manager(unit: str, records: List[_Record]) -> List[VirtualIpDict]: - values: Dict[str, Dict[str, str]] = {"arg": {}, "env": {}, "cfg": {}} +def _vip_manager(unit: str, records: list[_Record]) -> list[VirtualIpDict]: + values: dict[str, dict[str, str]] = {"arg": {}, "env": {}, "cfg": {}} for kind, rest in records: if kind in values: key, _, value = rest.partition(" ") values[kind][key] = value env = values["env"] - def resolve(value: str) -> Optional[str]: + def resolve(value: str) -> str | None: value = _clean(value) variable = _VARIABLE_RE.match(value) if variable: value = _clean(env.get(variable.group(1), "")) return value if value and "$" not in value else None - def pick(args: Tuple[str, ...], envs: Tuple[str, ...], cfgs: Tuple[str, ...]) -> Optional[str]: + def pick(args: tuple[str, ...], envs: tuple[str, ...], cfgs: tuple[str, ...]) -> str | None: for kind, names in (("arg", args), ("env", envs), ("cfg", cfgs)): for name in names: if name in values[kind]: @@ -272,15 +272,15 @@ def _vip_manager(unit: str, records: List[_Record]) -> List[VirtualIpDict]: @dataclass class _Instance: name: str - interface: Optional[str] = None + interface: str | None = None vrid: str = "" - vips: List[str] = field(default_factory=list) + vips: list[str] = field(default_factory=list) -def _keepalived(unit: str, records: List[_Record]) -> List[VirtualIpDict]: +def _keepalived(unit: str, records: list[_Record]) -> list[VirtualIpDict]: running = _running(records) - instances: List[_Instance] = [] - current: Optional[_Instance] = None + instances: list[_Instance] = [] + current: _Instance | None = None for kind, rest in records: if kind == "instance": current = _Instance(rest or unit) @@ -298,8 +298,8 @@ def _keepalived(unit: str, records: List[_Record]) -> List[VirtualIpDict]: return [entry for instance in instances for entry in _instance_entries(instance, running)] -def _instance_entries(instance: _Instance, running: Optional[bool]) -> List[VirtualIpDict]: - entries: List[VirtualIpDict] = [] +def _instance_entries(instance: _Instance, running: bool | None) -> list[VirtualIpDict]: + entries: list[VirtualIpDict] = [] for vip in instance.vips: tokens = vip.split() parsed = _address(tokens[0]) if tokens else None @@ -326,13 +326,13 @@ def _host_prefix(address: str) -> int: return ipaddress.ip_address(address).max_prefixlen -_READERS: Dict[str, Tuple[str, Callable[[str, List[_Record]], List[VirtualIpDict]]]] = { +_READERS: dict[str, tuple[str, Callable[[str, list[_Record]], list[VirtualIpDict]]]] = { "vip-manager": ("vip_manager", _vip_manager), "keepalived": ("keepalived", _keepalived), } -def _order(entry: VirtualIpDict) -> Tuple[int, int, str]: +def _order(entry: VirtualIpDict) -> tuple[int, int, str]: address = ipaddress.ip_address(entry["address"]) return address.version, int(address), entry["instance"] @@ -346,8 +346,8 @@ def parse_virtual_ips(output: str) -> VirtualIpsDict: sections = _sections(_frame(output)) if any(mechanism == _NO_SYSTEMD for mechanism, _, _ in sections): return {"vip_manager": None, "keepalived": None} - found: Dict[str, List[VirtualIpDict]] = {"vip_manager": [], "keepalived": []} - blind: Set[str] = set() + found: dict[str, list[VirtualIpDict]] = {"vip_manager": [], "keepalived": []} + blind: set[str] = set() for mechanism, unit, records in sections: if mechanism not in _READERS: continue @@ -356,7 +356,8 @@ def parse_virtual_ips(output: str) -> VirtualIpsDict: blind.add(key) else: found[key].extend(read(unit, records)) - def result(key: str) -> Optional[List[VirtualIpDict]]: + + def result(key: str) -> list[VirtualIpDict] | None: return None if key in blind else sorted(found[key], key=_order) return {"vip_manager": result("vip_manager"), "keepalived": result("keepalived")} diff --git a/pyproject.toml b/pyproject.toml index 8b0471a..6de333c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,13 @@ include = ["napalm_device_types*"] [tool.setuptools.package-data] napalm_device_types = ["py.typed"] +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] + [tool.mypy] python_version = "3.10" strict = true diff --git a/tests/test_channel.py b/tests/test_channel.py index 83d23c2..954237a 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -8,8 +8,6 @@ paramiko channel is faked, so the tests pin behaviour, not paramiko. from __future__ import annotations -import socket - import pytest from napalm_device_types.channel import ( @@ -56,7 +54,7 @@ class FakeChannel: if self.out: return self.out.pop(0) if self.hang: - raise socket.timeout() + raise TimeoutError() return b"" def recv_stderr_ready(self): diff --git a/tests/test_container_engine.py b/tests/test_container_engine.py index 53199a0..24db7da 100644 --- a/tests/test_container_engine.py +++ b/tests/test_container_engine.py @@ -32,7 +32,9 @@ class FakeDriver(ContainerEngineMixin): def run_command(self, command, *, privileged=False, timeout=60, stdin=None): self.commands.append((command, privileged, timeout, stdin)) found = any(command == f"command -v {b}" for b in self._binaries()) - return CommandResult(stdout="/usr/bin/x\n" if found else "", stderr="", exit_code=0 if found else 1) + return CommandResult( + stdout="/usr/bin/x\n" if found else "", stderr="", exit_code=0 if found else 1 + ) def open_stream(self, command, *, privileged=False): self.streams.append((command, privileged)) @@ -86,7 +88,9 @@ def test_cli_arguments_are_quoted_and_prefixed_by_the_binary(): def test_cli_stdin_and_timeout_reach_the_channel(): driver = FakeDriver() - driver.open_container_engine("docker").run_cli(["login", "--password-stdin"], stdin=b"x", timeout=30) + driver.open_container_engine("docker").run_cli( + ["login", "--password-stdin"], stdin=b"x", timeout=30 + ) _command, _privileged, timeout, stdin = driver.commands[-1] assert (timeout, stdin) == (30, b"x") diff --git a/tests/test_dhcp_diff_apply.py b/tests/test_dhcp_diff_apply.py index 2d567ca..4aaef44 100644 --- a/tests/test_dhcp_diff_apply.py +++ b/tests/test_dhcp_diff_apply.py @@ -9,9 +9,10 @@ principle: generic vs. device-specific logic" for why this logic lives here and not in a vendor driver. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import DhcpServerMixin, FirewallDriver, ResidentialGatewayDriver from napalm_device_types.dhcp import normalize_mac from napalm_device_types.models import DhcpReservationDict @@ -33,21 +34,21 @@ def _reservation(**overrides: Any) -> DhcpReservationDict: class _FakeDhcpServer(DhcpServerMixin): """In-memory fake — no network, no Kea/UCI specifics.""" - def __init__(self, live: Optional[List[DhcpReservationDict]] = None) -> None: + def __init__(self, live: list[DhcpReservationDict] | None = None) -> None: self.live = live or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.commits = 0 - def get_dhcp_reservations(self) -> List[DhcpReservationDict]: + def get_dhcp_reservations(self) -> list[DhcpReservationDict]: return self.live def apply_dhcp_reservation( - self, reservation: DhcpReservationDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"reservation": reservation, "uuid": uuid}) return {"success": True} - def commit_dhcp_reservations(self) -> Dict[str, Any]: + def commit_dhcp_reservations(self) -> dict[str, Any]: self.commits += 1 return {"success": True} diff --git a/tests/test_dhcp_ha_config_sync.py b/tests/test_dhcp_ha_config_sync.py index 89ccefa..fbabb79 100644 --- a/tests/test_dhcp_ha_config_sync.py +++ b/tests/test_dhcp_ha_config_sync.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Two facts an HA pair of servers reports about itself. `get_dhcp_ha` is what a DHCP server's own HA configuration says: the peers it diff --git a/tests/test_dhcp_subnet_diff_apply.py b/tests/test_dhcp_subnet_diff_apply.py index f478c42..39d3f1c 100644 --- a/tests/test_dhcp_subnet_diff_apply.py +++ b/tests/test_dhcp_subnet_diff_apply.py @@ -11,9 +11,10 @@ A subnet is a heavier object than a reservation: a wrong `pools` or below lean on the never-delete and preserve-unmanaged-options guarantees. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import DhcpServerMixin from napalm_device_types.models import DhcpSubnetDict @@ -35,21 +36,21 @@ def _subnet(**overrides: Any) -> DhcpSubnetDict: class _FakeDhcpServer(DhcpServerMixin): - def __init__(self, live: Optional[List[DhcpSubnetDict]] = None) -> None: + def __init__(self, live: list[DhcpSubnetDict] | None = None) -> None: self.live = live or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.commits = 0 - def get_dhcp_subnets(self) -> List[DhcpSubnetDict]: + def get_dhcp_subnets(self) -> list[DhcpSubnetDict]: return self.live def apply_dhcp_subnet( - self, subnet: DhcpSubnetDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, subnet: DhcpSubnetDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"subnet": subnet, "uuid": uuid}) return {"success": True} - def commit_dhcp_subnets(self) -> Dict[str, Any]: + def commit_dhcp_subnets(self) -> dict[str, Any]: self.commits += 1 return {"success": True} @@ -215,9 +216,7 @@ class _BareDriver(DhcpServerMixin): pass -@pytest.mark.parametrize( - "method", ["get_dhcp_subnets", "apply_dhcp_subnet", "commit_dhcp_subnets"] -) +@pytest.mark.parametrize("method", ["get_dhcp_subnets", "apply_dhcp_subnet", "commit_dhcp_subnets"]) def test_device_specific_methods_are_absent_until_implemented(method: str) -> None: """Declared under TYPE_CHECKING, so they do not exist until a driver adds them.""" assert not hasattr(_BareDriver, method) diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py index f715bfc..ac820fa 100644 --- a/tests/test_fact_getters.py +++ b/tests/test_fact_getters.py @@ -31,7 +31,7 @@ def test_a_missing_license_may_say_what_the_device_withholds_without_it(): # firmware inventory until DCMS is activated: partly licensed, not unlicensed. hints = get_type_hints(LicenseDict) # Python 3.10 keeps NotRequired[...] around the type; later versions strip it. - assert "List[str]" in str(hints["withholds"]) + assert "list[str]" in str(hints["withholds"]) assert "withholds" not in LicenseDict.__required_keys__ diff --git a/tests/test_fingerprint_base.py b/tests/test_fingerprint_base.py index 89738f9..3ad761d 100644 --- a/tests/test_fingerprint_base.py +++ b/tests/test_fingerprint_base.py @@ -3,21 +3,21 @@ from __future__ import annotations import pytest +from napalm.base import NetworkDriver + from napalm_device_types import ( AccessPointDriver, DeviceTypeDriver, FingerprintRule, FirewallDriver, - rules_max_weight, HypervisorDriver, OSDriver, PortSpec, ResidentialGatewayDriver, StorageDriver, SwitchDriver, + rules_max_weight, ) -from napalm.base import NetworkDriver - # ── Hierarchy ───────────────────────────────────────────────────────────────── @@ -49,6 +49,7 @@ def test_device_type_driver_inherits_network_driver(): # ── Default attributes ──────────────────────────────────────────────────────── + def test_defaults_on_device_type_driver(): assert DeviceTypeDriver.VENDOR == "" assert DeviceTypeDriver.DRIVER_NAME == "" @@ -72,6 +73,7 @@ def test_fingerprint_attributes_inherited(cls): # ── FingerprintRule ─────────────────────────────────────────────────────────── + def test_fingerprint_rule_defaults(): r = FingerprintRule("fritz!box") assert r.pattern == "fritz!box" @@ -102,6 +104,7 @@ def test_fingerprint_rule_group(): # ── rules_max_weight: the most a set of rules can add ───────────────────────── + def test_max_weight_sums_rules_that_can_match_together(): rules = [FingerprintRule("proxmox virtual environment", 9.0), FingerprintRule("pve", 2.0)] assert rules_max_weight(rules) == 11.0 @@ -146,6 +149,7 @@ def test_fingerprint_rule_is_immutable(): # ── PortSpec ────────────────────────────────────────────────────────────────── + def test_port_spec_defaults(): p = PortSpec("https", 8006) assert p.scheme == "https" @@ -169,6 +173,7 @@ def test_port_spec_is_immutable(): # ── Concrete driver subclass ────────────────────────────────────────────────── + def test_concrete_driver_overrides_fingerprint(): class MyDriver(FirewallDriver): VENDOR = "Acme" diff --git a/tests/test_firewall_diff_apply.py b/tests/test_firewall_diff_apply.py index 471972c..1001990 100644 --- a/tests/test_firewall_diff_apply.py +++ b/tests/test_firewall_diff_apply.py @@ -9,9 +9,10 @@ device-specific logic" for why this logic lives here and not in a vendor driver. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import FirewallDriver from napalm_device_types.models import FirewallRuleDict @@ -39,21 +40,21 @@ def _rule(**overrides: Any) -> FirewallRuleDict: class _FakeFirewall(FirewallDriver): """In-memory fake — no network, no OPNsense/vendor specifics.""" - def __init__(self, live_rules: Optional[List[FirewallRuleDict]] = None) -> None: + def __init__(self, live_rules: list[FirewallRuleDict] | None = None) -> None: self.live_rules = live_rules or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.committed = False - def get_firewall_rules(self) -> List[FirewallRuleDict]: + def get_firewall_rules(self) -> list[FirewallRuleDict]: return self.live_rules def apply_firewall_rule( - self, rule: FirewallRuleDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, rule: FirewallRuleDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"rule": rule, "uuid": uuid}) return {"success": True} - def commit_firewall_rules(self) -> Dict[str, Any]: + def commit_firewall_rules(self) -> dict[str, Any]: self.committed = True return {"success": True} diff --git a/tests/test_fix_templates.py b/tests/test_fix_templates.py index 886a975..e6b866f 100644 --- a/tests/test_fix_templates.py +++ b/tests/test_fix_templates.py @@ -17,12 +17,12 @@ from napalm_device_types import ( ManagementControllerDriver, MediaDriver, OSDriver, + PackageProxyMixin, PhoneDriver, ResidentialGatewayDriver, + SnmpAgentMixin, StorageDriver, SwitchDriver, - PackageProxyMixin, - SnmpAgentMixin, run_fix_steps, ) @@ -97,7 +97,12 @@ class _Agent(SnmpAgentMixin): class TestSnmpAgent: - SETTINGS = {"community": "c0mm", "allowed_sources": ["10.0.0.5"], "contact": "ops", "location": "rack 1"} + SETTINGS = { + "community": "c0mm", + "allowed_sources": ["10.0.0.5"], + "contact": "ops", + "location": "rack 1", + } def test_install_configure_open_and_start_with_the_caller_s_values(self): agent = _Agent() @@ -153,7 +158,10 @@ class TestEnsurePackage: def test_an_install_that_reports_failure_fails(self): driver = _Packages(installed=[]) - driver.install_package = lambda name, version="": {"success": False, "output": "E: no candidate"} + driver.install_package = lambda name, version="": { + "success": False, + "output": "E: no candidate", + } result = driver.ensure_package("lldpd") @@ -164,7 +172,12 @@ class TestEnsurePackage: @pytest.mark.parametrize("base", ROLE_BASES, ids=lambda b: b.__name__) @pytest.mark.parametrize( "method", - ["configure_snmp_agent", "ensure_package", "configure_package_proxy", "grant_container_engine_access"], + [ + "configure_snmp_agent", + "ensure_package", + "configure_package_proxy", + "grant_container_engine_access", + ], ) def test_no_role_base_offers_a_fix(base, method): assert getattr(base, method, None) is None diff --git a/tests/test_host_status.py b/tests/test_host_status.py index 1b3341c..2465e46 100644 --- a/tests/test_host_status.py +++ b/tests/test_host_status.py @@ -452,7 +452,10 @@ class TestBusyCheck: UBUNTU_IDLE = [ ("systemd", "/sbin/init"), - ("unattended-upgr", "/usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal"), + ( + "unattended-upgr", + "/usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal", + ), ("sshd", "sshd: netork [priv]"), ] diff --git a/tests/test_lag_interfaces.py b/tests/test_lag_interfaces.py index 852655c..f144936 100644 --- a/tests/test_lag_interfaces.py +++ b/tests/test_lag_interfaces.py @@ -3,7 +3,9 @@ from napalm_device_types import add_lag_interfaces -def _port(is_up: bool = True, is_enabled: bool = True, speed: float = 1000.0, trunk_group: str = "") -> dict: +def _port( + is_up: bool = True, is_enabled: bool = True, speed: float = 1000.0, trunk_group: str = "" +) -> dict: port = { "is_up": is_up, "is_enabled": is_enabled, diff --git a/tests/test_listening_sockets.py b/tests/test_listening_sockets.py index b67954f..e808637 100644 --- a/tests/test_listening_sockets.py +++ b/tests/test_listening_sockets.py @@ -89,7 +89,7 @@ class TestParsing: ], ) def test_every_address_form(self, local: str, address: str, interface): - line = f"tcp LISTEN 0 128 {local} *:* users:((\"x\",pid=1,fd=3))\n" + line = f'tcp LISTEN 0 128 {local} *:* users:(("x",pid=1,fd=3))\n' [socket] = parse_listening_sockets(_wire(line, "1 0::/system.slice/x.service\n")) @@ -188,7 +188,7 @@ class TestFailures: def test_a_report_cut_short_raises(self): with pytest.raises(ValueError): - parse_listening_sockets(_wire()[:-len("SOCK_END\n")]) + parse_listening_sockets(_wire()[: -len("SOCK_END\n")]) def test_a_host_without_ss_or_netstat_is_unavailable(self): with pytest.raises(ListeningSocketsUnavailable): diff --git a/tests/test_ping_sweep.py b/tests/test_ping_sweep.py index 90bef7d..03a34db 100644 --- a/tests/test_ping_sweep.py +++ b/tests/test_ping_sweep.py @@ -5,7 +5,6 @@ from napalm.base import NetworkDriver from napalm_device_types import DeviceTypeDriver, PingSweepMixin, driver_supports_ping - # ── Fakes ───────────────────────────────────────────────────────────────────── @@ -225,4 +224,9 @@ def test_ping_sweep_stops_when_stop_requested(): def test_a_malformed_success_is_no_answer_not_an_abort(success): entry = PingSweepMixin._parse_ping_reply("192.0.2.1", {"success": success}) - assert entry == {"ip": "192.0.2.1", "alive": False, "rtt_ms": None, "error": "malformed ping reply"} + assert entry == { + "ip": "192.0.2.1", + "alive": False, + "rtt_ms": None, + "error": "malformed ping reply", + } diff --git a/tests/test_port_forwards.py b/tests/test_port_forwards.py index 7807d39..19b3ccb 100644 --- a/tests/test_port_forwards.py +++ b/tests/test_port_forwards.py @@ -19,7 +19,7 @@ from napalm_device_types.nat_vpn import NatVpnMixin def test_a_firewall_and_a_gateway_share_the_declaration(): assert issubclass(FirewallDriver, NatVpnMixin) assert issubclass(ResidentialGatewayDriver, NatVpnMixin) - assert "def get_port_forwards(self) -> List[PortForwardDict]" in inspect.getsource(NatVpnMixin) + assert "def get_port_forwards(self) -> list[PortForwardDict]" in inspect.getsource(NatVpnMixin) def test_it_is_declared_once(): diff --git a/tests/test_provisioning.py b/tests/test_provisioning.py index dd52b73..56ff10f 100644 --- a/tests/test_provisioning.py +++ b/tests/test_provisioning.py @@ -4,6 +4,7 @@ the same for every hypervisor (NetOrk/netork#794).""" from __future__ import annotations import pytest + from napalm_device_types import HypervisorDriver from napalm_device_types.provisioning import ( QEMU_GUEST_AGENTS, diff --git a/tests/test_role_contracts.py b/tests/test_role_contracts.py index a8b9f50..0c60958 100644 --- a/tests/test_role_contracts.py +++ b/tests/test_role_contracts.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The contract that makes multi-role drivers safe. A role base declares what a device of that kind can be asked for. It must not diff --git a/tests/test_send_wake_on_lan.py b/tests/test_send_wake_on_lan.py index b392ffd..128855c 100644 --- a/tests/test_send_wake_on_lan.py +++ b/tests/test_send_wake_on_lan.py @@ -9,6 +9,7 @@ sibling base. """ import pytest + from napalm_device_types import FirewallDriver diff --git a/tests/test_service_clusters.py b/tests/test_service_clusters.py index 2bf8a7f..9c74810 100644 --- a/tests/test_service_clusters.py +++ b/tests/test_service_clusters.py @@ -35,14 +35,29 @@ SECRET = "s3cr3t-do-not-leak" def _wire(*sections: str, systemd: bool = True, docker: str = "[docker]", noise: str = "") -> str: head = "[systemd]" if systemd else "[no-systemd]" - body = "\n".join([head, *sections[:1], docker, *sections[1:]]) if sections else f"{head}\n{docker}" + body = ( + "\n".join([head, *sections[:1], docker, *sections[1:]]) if sections else f"{head}\n{docker}" + ) return f"{noise}SVCC_BEGIN\n{body}\nSVCC_END\n" DB01_CLUSTER = { "members": [ - {"name": "db-01", "role": "leader", "state": "running", "host": "10.7.234.12", "port": 5432}, - {"name": "db-02", "role": "sync_standby", "state": "streaming", "host": "10.7.234.11", "port": 5432, "lag": 0}, + { + "name": "db-01", + "role": "leader", + "state": "running", + "host": "10.7.234.12", + "port": 5432, + }, + { + "name": "db-02", + "role": "sync_standby", + "state": "streaming", + "host": "10.7.234.11", + "port": 5432, + "lag": 0, + }, ], "scope": "netork-db", } @@ -79,10 +94,24 @@ class TestPatroni: "cluster": "netork-db", "running": True, "members": [ - {"name": "db-01", "address": "10.7.234.12", "port": 5432, "role": "leader", - "state": "running", "this_host": True, "config_from": None}, - {"name": "db-02", "address": "10.7.234.11", "port": 5432, "role": "sync_standby", - "state": "streaming", "this_host": False, "config_from": None}, + { + "name": "db-01", + "address": "10.7.234.12", + "port": 5432, + "role": "leader", + "state": "running", + "this_host": True, + "config_from": None, + }, + { + "name": "db-02", + "address": "10.7.234.11", + "port": 5432, + "role": "sync_standby", + "state": "streaming", + "this_host": False, + "config_from": None, + }, ], } ] @@ -91,8 +120,11 @@ class TestPatroni: reading = parse_service_clusters( _wire( _patroni( - "active active", "scope 'netork-db' # the cluster", "name db-01", - "restapi 10.7.234.12:8008", "rest-unreachable", + "active active", + "scope 'netork-db' # the cluster", + "name db-01", + "restapi 10.7.234.12:8008", + "rest-unreachable", ) ) ) @@ -100,8 +132,15 @@ class TestPatroni: (cluster,) = reading["patroni"] assert cluster["cluster"] == "netork-db" assert cluster["members"] == [ - {"name": "db-01", "address": "10.7.234.12", "port": None, "role": None, - "state": None, "this_host": True, "config_from": None} + { + "name": "db-01", + "address": "10.7.234.12", + "port": None, + "role": None, + "state": None, + "this_host": True, + "config_from": None, + } ] def test_an_unreadable_config_is_not_looked_at(self): @@ -112,7 +151,9 @@ class TestPatroni: assert reading["patroni"] is None def test_a_config_without_a_scope_declares_no_cluster(self): - assert parse_service_clusters(_wire(_patroni("active active", "name db-01")))["patroni"] == [] + assert ( + parse_service_clusters(_wire(_patroni("active active", "name db-01")))["patroni"] == [] + ) def test_a_host_without_systemd_is_not_looked_at(self): assert parse_service_clusters(_wire(systemd=False))["patroni"] is None @@ -127,8 +168,10 @@ class TestAdguardhomeSync: _wire( "", _container( - "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3", - "ORIGIN_URL http://10.9.100.5:3000", "REPLICA1_URL http://10.9.100.6:3000", + "adguardhome-sync", + "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL http://10.9.100.5:3000", + "REPLICA1_URL http://10.9.100.6:3000", "REPLICA2_URL https://dns-03.example.org", ), ) @@ -136,9 +179,13 @@ class TestAdguardhomeSync: (cluster,) = reading["adguardhome_sync"] assert (cluster["mechanism"], cluster["service"], cluster["cluster"]) == ( - "adguardhome-sync", "adguardhome", "10.9.100.5:3000", + "adguardhome-sync", + "adguardhome", + "10.9.100.5:3000", ) - assert [(m["address"], m["port"], m["role"], m["config_from"]) for m in cluster["members"]] == [ + assert [ + (m["address"], m["port"], m["role"], m["config_from"]) for m in cluster["members"] + ] == [ ("10.9.100.5", 3000, "origin", None), ("10.9.100.6", 3000, "replica", 0), ("dns-03.example.org", 443, "replica", 0), @@ -147,8 +194,15 @@ class TestAdguardhomeSync: def test_a_single_replica_url_counts(self): reading = parse_service_clusters( - _wire("", _container("sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000", - "REPLICA_URL http://b:3000")) + _wire( + "", + _container( + "sync", + "bakito/adguardhome-sync", + "ORIGIN_URL http://a:3000", + "REPLICA_URL http://b:3000", + ), + ) ) assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == ["a", "b"] @@ -163,36 +217,68 @@ class TestAdguardhomeSync: class TestBunkerweb: def test_a_worker_takes_its_configuration_from_the_manager_it_admits(self): reading = parse_service_clusters( - _wire("", _container("bunkerweb", "bunkerity/bunkerweb@sha256:2026", "API_WHITELIST_IP 10.50.100.5")) + _wire( + "", + _container( + "bunkerweb", "bunkerity/bunkerweb@sha256:2026", "API_WHITELIST_IP 10.50.100.5" + ), + ) ) (cluster,) = reading["bunkerweb"] assert (cluster["service"], cluster["cluster"]) == ("bunkerweb", "10.50.100.5") assert cluster["members"] == [ - {"name": None, "address": None, "port": 5000, "role": "worker", "state": None, - "this_host": True, "config_from": 1}, - {"name": None, "address": "10.50.100.5", "port": None, "role": "manager", "state": None, - "this_host": False, "config_from": None}, + { + "name": None, + "address": None, + "port": 5000, + "role": "worker", + "state": None, + "this_host": True, + "config_from": 1, + }, + { + "name": None, + "address": "10.50.100.5", + "port": None, + "role": "manager", + "state": None, + "this_host": False, + "config_from": None, + }, ] def test_the_default_whitelist_names_no_manager(self): """127.0.0.0/8 and a docker network are BunkerWeb's default, not a peer.""" reading = parse_service_clusters( - _wire("", _container("bw", "bunkerity/bunkerweb:1.6", "API_WHITELIST_IP 127.0.0.0/8 10.20.30.0/24")) + _wire( + "", + _container( + "bw", "bunkerity/bunkerweb:1.6", "API_WHITELIST_IP 127.0.0.0/8 10.20.30.0/24" + ), + ) ) assert reading["bunkerweb"] == [] def test_a_manager_names_its_instances(self): reading = parse_service_clusters( - _wire("", _container("bw", "bunkerity/bunkerweb-all-in-one:1.6", - "BUNKERWEB_INSTANCES 10.50.100.6:5000 bw-03")), + _wire( + "", + _container( + "bw", + "bunkerity/bunkerweb-all-in-one:1.6", + "BUNKERWEB_INSTANCES 10.50.100.6:5000 bw-03", + ), + ), ) (cluster,) = reading["bunkerweb"] assert cluster["cluster"] is None # the manager cannot name its own address - assert [(m["address"], m["port"], m["role"], m["this_host"], m["config_from"]) - for m in cluster["members"]] == [ + assert [ + (m["address"], m["port"], m["role"], m["this_host"], m["config_from"]) + for m in cluster["members"] + ] == [ (None, None, "manager", True, None), ("10.50.100.6", 5000, "worker", False, 0), ("bw-03", 5000, "worker", False, 0), @@ -200,7 +286,9 @@ class TestBunkerweb: def test_a_manager_without_instances_is_no_cluster(self): """proxy-01's all-in-one keeps its instances in its database.""" - reading = parse_service_clusters(_wire("", _container("bw", "bunkerity/bunkerweb-all-in-one@sha256:2b34"))) + reading = parse_service_clusters( + _wire("", _container("bw", "bunkerity/bunkerweb-all-in-one@sha256:2b34")) + ) assert reading["bunkerweb"] == [] @@ -298,7 +386,7 @@ class _Host: "docker", 'case "$1" in\n' f' ps) {cat} "$d/containers" ;;\n' - ' inspect) for last; do :; done;\n' + " inspect) for last; do :; done;\n" ' case "$3" in *Config.Image*) ' f'{cat} "$d/image-$last" ;; *) {cat} "$d/env-$last" ;; esac ;;\n' "esac\n", @@ -362,7 +450,9 @@ def host(request, tmp_path) -> _Host: class TestTheCommandOnAHost: def test_patroni_as_on_db_01(self, host): config = host.file("etc/patroni/config.yml", PATRONI_YML) - host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) host.rest(DB01_CLUSTER) out = host.run() @@ -379,7 +469,9 @@ class TestTheCommandOnAHost: def test_patroni_whose_rest_api_does_not_answer(self, host): config = host.file("etc/patroni/config.yml", PATRONI_YML) - host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) (cluster,) = parse_service_clusters(host.run())["patroni"] @@ -391,7 +483,9 @@ class TestTheCommandOnAHost: def test_a_patroni_config_it_cannot_read_is_not_looked_at(self, host): config = host.file("etc/patroni/config.yml", PATRONI_YML) config.chmod(0) - host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config))) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) out = host.run() @@ -400,14 +494,19 @@ class TestTheCommandOnAHost: def test_the_sync_and_a_worker_without_their_secrets(self, host): host.container( - "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3", - "ORIGIN_URL=http://10.9.100.5:3000", f"ORIGIN_PASSWORD={SECRET}", - f"REPLICA1_URL=http://admin:{SECRET}@10.9.100.6:3000", f"REPLICA1_PASSWORD={SECRET}", + "adguardhome-sync", + "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL=http://10.9.100.5:3000", + f"ORIGIN_PASSWORD={SECRET}", + f"REPLICA1_URL=http://admin:{SECRET}@10.9.100.6:3000", + f"REPLICA1_PASSWORD={SECRET}", "CRON=*/5 * * * *", ) host.container( - "bunkerweb", "bunkerity/bunkerweb@sha256:2026", - "API_WHITELIST_IP=10.50.100.5", f"DATABASE_URI=mariadb+pymysql://bw:{SECRET}@db/bw", + "bunkerweb", + "bunkerity/bunkerweb@sha256:2026", + "API_WHITELIST_IP=10.50.100.5", + f"DATABASE_URI=mariadb+pymysql://bw:{SECRET}@db/bw", ) host.container("zabbix", "zabbix/zabbix-agent2:alpine-latest", f"ZBX_PSK={SECRET}") @@ -417,7 +516,8 @@ class TestTheCommandOnAHost: assert "admin" not in out reading = parse_service_clusters(out) assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == [ - "10.9.100.5", "10.9.100.6", + "10.9.100.5", + "10.9.100.6", ] assert reading["bunkerweb"][0]["cluster"] == "10.50.100.5" assert reading["patroni"] == [] @@ -483,8 +583,13 @@ class TestTheSource: def test_a_container_by_its_name(self): reading = parse_service_clusters( - _wire("", _container("adguardhome-sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000"), - _container("bw", "bunkerity/bunkerweb", "API_WHITELIST_IP 10.0.0.5")) + _wire( + "", + _container( + "adguardhome-sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000" + ), + _container("bw", "bunkerity/bunkerweb", "API_WHITELIST_IP 10.0.0.5"), + ) ) assert reading["adguardhome_sync"][0]["source"] == "adguardhome-sync" diff --git a/tests/test_ssh_host_keys.py b/tests/test_ssh_host_keys.py index 0a56bee..f2d91a8 100644 --- a/tests/test_ssh_host_keys.py +++ b/tests/test_ssh_host_keys.py @@ -261,7 +261,9 @@ class TestNetmiko: def test_any_other_failure_passes_unchanged(self): with pytest.raises(ConnectionException, match="timed out"): - with HostKeyCheck({KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}}, "10.0.0.5", 22).netmiko({}): + with HostKeyCheck({KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}}, "10.0.0.5", 22).netmiko( + {} + ): raise ConnectionException("timed out") def test_observe_reports_the_key_netmiko_saw(self, host_key): diff --git a/tests/test_ssh_server.py b/tests/test_ssh_server.py index 462cea9..ad1db25 100644 --- a/tests/test_ssh_server.py +++ b/tests/test_ssh_server.py @@ -9,7 +9,6 @@ cannot get through -- runs without a machine. from __future__ import annotations import re -from typing import Dict, List, Optional import pytest @@ -69,7 +68,11 @@ def test_parse_reads_the_four_lists_in_server_order(): eff = parse_sshd_effective(SSHD_T) assert eff["kex"][:2] == ["sntrup761x25519-sha512@openssh.com", "curve25519-sha256"] assert eff["hostkey"] == ["ssh-ed25519", "ecdsa-sha2-nistp256", "rsa-sha2-512", "rsa-sha2-256"] - assert eff["cipher"] == ["chacha20-poly1305@openssh.com", "aes128-ctr", "aes256-gcm@openssh.com"] + assert eff["cipher"] == [ + "chacha20-poly1305@openssh.com", + "aes128-ctr", + "aes256-gcm@openssh.com", + ] assert eff["mac"][-1] == "hmac-sha2-256" @@ -82,7 +85,9 @@ def test_parse_refuses_output_without_the_lists(): def test_plan_removes_what_was_asked_and_keeps_the_order(): - plan = plan_ssh_hardening(parse_sshd_effective(SSHD_T), {"mac": ["hmac-sha1", "umac-64-etm@openssh.com"]}) + plan = plan_ssh_hardening( + parse_sshd_effective(SSHD_T), {"mac": ["hmac-sha1", "umac-64-etm@openssh.com"]} + ) assert plan.remove == {"mac": ["umac-64-etm@openssh.com", "hmac-sha1"]} assert plan.keep["mac"] == ["hmac-sha2-256-etm@openssh.com", "hmac-sha2-256"] assert plan.changed_classes == ["mac"] @@ -113,20 +118,38 @@ def test_plan_keeps_one_algorithm_the_managing_client_still_speaks(): know would leave the server reachable, but not by netOrk.""" plan = plan_ssh_hardening( parse_sshd_effective(SSHD_T), - {"kex": ["curve25519-sha256@libssh.org", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", - "diffie-hellman-group14-sha256"]}, + { + "kex": [ + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] + }, client=PARAMIKO, ) assert "curve25519-sha256@libssh.org" in plan.keep["kex"] - assert [(r.algorithm, r.reason) for r in plan.refused] == [("curve25519-sha256@libssh.org", "client")] - assert plan.remove["kex"] == ["ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "diffie-hellman-group14-sha256"] + assert [(r.algorithm, r.reason) for r in plan.refused] == [ + ("curve25519-sha256@libssh.org", "client") + ] + assert plan.remove["kex"] == [ + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] def test_plan_without_a_client_list_only_guards_the_last_algorithm(): plan = plan_ssh_hardening( parse_sshd_effective(SSHD_T), - {"kex": ["curve25519-sha256@libssh.org", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", - "diffie-hellman-group14-sha256"]}, + { + "kex": [ + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] + }, ) assert plan.refused == [] assert plan.keep["kex"] == ["sntrup761x25519-sha512@openssh.com", "curve25519-sha256"] @@ -171,12 +194,18 @@ def test_plan_removes_a_host_key_algorithm_when_a_usable_one_stays(): def test_plan_leaves_host_keys_alone_when_the_keys_are_unknown(): plan = plan_ssh_hardening(parse_sshd_effective(SSHD_T), {"hostkey": ["ecdsa-sha2-nistp256"]}) assert plan.changed_classes == [] - assert [(r.algorithm, r.reason) for r in plan.refused] == [("ecdsa-sha2-nistp256", "host_keys_unknown")] + assert [(r.algorithm, r.reason) for r in plan.refused] == [ + ("ecdsa-sha2-nistp256", "host_keys_unknown") + ] def test_host_key_algorithms_from_ssh_keygen(): assert host_key_algorithms("\n".join(KEYGEN.values())) == { - "ecdsa-sha2-nistp256", "ssh-ed25519", "rsa-sha2-512", "rsa-sha2-256", "ssh-rsa", + "ecdsa-sha2-nistp256", + "ssh-ed25519", + "rsa-sha2-512", + "rsa-sha2-256", + "ssh-rsa", } assert host_key_algorithms("521 SHA256:x no comment (ECDSA)") == {"ecdsa-sha2-nistp521"} assert host_key_algorithms("") is None @@ -204,11 +233,15 @@ def test_include_is_found_only_in_the_global_section(): assert has_dropin_include("Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n") assert has_dropin_include(" include /etc/ssh/sshd_config.d/*.conf\n") assert not has_dropin_include("Port 22\n") - assert not has_dropin_include("Port 22\nMatch User x\n Include /etc/ssh/sshd_config.d/*.conf\n") + assert not has_dropin_include( + "Port 22\nMatch User x\n Include /etc/ssh/sshd_config.d/*.conf\n" + ) def test_main_config_gets_the_lists_first_and_the_old_lines_disabled(): - main = "# defaults\nPort 22\nMACs hmac-sha1,hmac-sha2-256\nMatch User backup\n MACs hmac-sha1\n" + main = ( + "# defaults\nPort 22\nMACs hmac-sha1,hmac-sha2-256\nMatch User backup\n MACs hmac-sha1\n" + ) text = render_main_config(main, {"mac": ["hmac-sha2-256"]}, ["mac"]) lines = text.splitlines() first_directive = next(line for line in lines if line and not line.startswith("#")) @@ -227,28 +260,34 @@ def test_main_config_rewrite_is_idempotent(): # ── the mixin, against a fake host ─────────────────────────────────────── -_DIRECTIVE = {"kexalgorithms": "kex", "hostkeyalgorithms": "hostkey", "ciphers": "cipher", "macs": "mac"} +_DIRECTIVE = { + "kexalgorithms": "kex", + "hostkeyalgorithms": "hostkey", + "ciphers": "cipher", + "macs": "mac", +} class FakeHost(OpenSshServerMixin): """A host with files, an sshd that reads them first-match-wins, and a log of what ran.""" - def __init__(self, main: str = "Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n", - sshd_t: str = SSHD_T) -> None: - self.files: Dict[str, str] = {MAIN_CONFIG: main} - self.modes: Dict[str, str] = {MAIN_CONFIG: "644"} + def __init__( + self, main: str = "Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n", sshd_t: str = SSHD_T + ) -> None: + self.files: dict[str, str] = {MAIN_CONFIG: main} + self.modes: dict[str, str] = {MAIN_CONFIG: "644"} self.sshd_t = sshd_t self.defaults = parse_sshd_effective(sshd_t) self.keys = {p: KEYGEN[p] for p in parse_sshd_hostkeys(sshd_t)} self.keygen_ok = True - self.ran: List[str] = [] + self.ran: list[str] = [] self.syntax_ok = True self.reloads = 0 - def _effective(self) -> Dict[str, List[str]]: + def _effective(self) -> dict[str, list[str]]: eff = {k: list(v) for k, v in self.defaults.items()} seen = set() - lines: List[str] = [] + lines: list[str] = [] for line in self.files[MAIN_CONFIG].splitlines(): if re.match(r"\s*include\s", line, re.I): lines += self.files.get(DROPIN_PATH, "").splitlines() @@ -269,8 +308,14 @@ class FakeHost(OpenSshServerMixin): #: or a cached timestamp): the password the driver put in front of the data. SUDO_PASSWORD = "hunter2" - def run_command(self, command: str, *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None) -> CommandResult: + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: self.ran.append(command) if privileged and stdin is not None: stdin = f"{self.SUDO_PASSWORD}\n".encode() + stdin @@ -294,14 +339,20 @@ class FakeHost(OpenSshServerMixin): if not self.keygen_ok: return CommandResult("", "Permission denied", 1) paths = re.search(r"for k in (.*?); do", command).group(1).replace("'", "").split() - return CommandResult("".join(self.keys[p] + "\n" for p in paths if p in self.keys), "", 0) + return CommandResult( + "".join(self.keys[p] + "\n" for p in paths if p in self.keys), "", 0 + ) if command.startswith("sshd -T") or "sshd -T" in command: eff = self._effective() out = "".join(f"{d} {','.join(eff[k])}\n" for d, k in _DIRECTIVE.items()) - out += "".join(line + "\n" for line in self.sshd_t.splitlines() if line.startswith("hostkey ")) + out += "".join( + line + "\n" for line in self.sshd_t.splitlines() if line.startswith("hostkey ") + ) return CommandResult(out, "", 0) if "sshd -t" in command: - return CommandResult("", "" if self.syntax_ok else "bad line 3", 0 if self.syntax_ok else 255) + return CommandResult( + "", "" if self.syntax_ok else "bad line 3", 0 if self.syntax_ok else 255 + ) m = re.match(r"cat -- '?([^' ]+)'?$", command) if m: path = m.group(1) @@ -323,7 +374,7 @@ class FakeHost(OpenSshServerMixin): def test_mixin_writes_the_dropin_checks_it_and_reloads(): host = FakeHost() - lines: List[str] = [] + lines: list[str] = [] result = host.harden_ssh_server({"mac": ["hmac-sha1"]}, client=PARAMIKO, log=lines.append) assert result.applied and not result.rolled_back assert result.path == DROPIN_PATH @@ -439,7 +490,7 @@ def test_mixin_reports_a_failed_way_back_instead_of_raising(): def test_mixin_refuses_to_lock_out_a_host_with_only_ed25519_and_rsa_keys(): host = FakeHost(sshd_t=SSHD_T_FEW_KEYS) - lines: List[str] = [] + lines: list[str] = [] result = host.harden_ssh_server( {"hostkey": ["ssh-ed25519", "rsa-sha2-256", "rsa-sha2-512"]}, log=lines.append ) diff --git a/tests/test_system_identity.py b/tests/test_system_identity.py index 2dc8d8b..c874b00 100644 --- a/tests/test_system_identity.py +++ b/tests/test_system_identity.py @@ -38,8 +38,20 @@ def test_a_supermicro_placeholder_serial_is_dropped(): @pytest.mark.parametrize( "value", - ["", "None", "N/A", "Not Specified", "To be filled by O.E.M.", "Default string", - "System Serial Number", "0123456789", "1234567890", "0", "00000000", "Not Applicable"], + [ + "", + "None", + "N/A", + "Not Specified", + "To be filled by O.E.M.", + "Default string", + "System Serial Number", + "0123456789", + "1234567890", + "0", + "00000000", + "Not Applicable", + ], ) def test_placeholders(value): assert is_placeholder_identity(value) @@ -47,9 +59,12 @@ def test_placeholders(value): @pytest.mark.parametrize( "uuid", - ["00000000-0000-0000-0000-000000000000", "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", - # A UUID many cheap boards ship, the same on every one of them. - "03000200-0400-0500-0006-000700080009"], + [ + "00000000-0000-0000-0000-000000000000", + "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", + # A UUID many cheap boards ship, the same on every one of them. + "03000200-0400-0500-0006-000700080009", + ], ) def test_placeholder_uuids(uuid): assert parse_system_identity(_out(uuid, "", ""))["system_uuid"] == "" @@ -63,7 +78,10 @@ def test_real_values_are_not_placeholders(): def test_unreadable_files_read_as_empty(): # product_serial and product_uuid are root's alone; without root they come back empty. assert parse_system_identity(_out("", "", "BS1")) == { - "system_uuid": "", "product_serial": "", "board_serial": "BS1"} + "system_uuid": "", + "product_serial": "", + "board_serial": "BS1", + } def test_output_without_its_frame_is_not_a_reading(): diff --git a/tests/test_systemd_services.py b/tests/test_systemd_services.py index 9423baf..7d9695b 100644 --- a/tests/test_systemd_services.py +++ b/tests/test_systemd_services.py @@ -409,4 +409,4 @@ class TestSystemdServicesMixin: def test_not_every_os_driver_has_it(self): assert not hasattr(OSDriver, "manage_service") - assert callable(getattr(SystemdServicesMixin, "manage_service")) + assert callable(SystemdServicesMixin.manage_service) diff --git a/tests/test_type_label.py b/tests/test_type_label.py index c949a5e..c215788 100644 --- a/tests/test_type_label.py +++ b/tests/test_type_label.py @@ -1,6 +1,7 @@ """Tests for TYPE_LABEL class attribute on all base driver classes.""" import pytest + from napalm_device_types import ( AccessPointDriver, FirewallDriver, @@ -12,7 +13,6 @@ from napalm_device_types import ( SwitchDriver, ) - BASE_CLASSES = [ (AccessPointDriver, "Access Point"), (FirewallDriver, "Firewall"), diff --git a/tests/test_ucd_metrics.py b/tests/test_ucd_metrics.py index eec7614..14fcadf 100644 --- a/tests/test_ucd_metrics.py +++ b/tests/test_ucd_metrics.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -from typing import Dict, Optional from napalm_device_types._ucd_metrics import ( OID_IF_DESCR, @@ -24,11 +23,11 @@ LEGACY = { } -def _collect(tables: Dict[str, Dict[str, str]]) -> dict: - async def get(oid: str) -> Optional[str]: +def _collect(tables: dict[str, dict[str, str]]) -> dict: + async def get(oid: str) -> str | None: return None - async def walk(oid: str) -> Dict[str, str]: + async def walk(oid: str) -> dict[str, str]: return tables.get(oid, {}) return asyncio.run(collect_ucd_metrics(get, walk)) @@ -44,7 +43,11 @@ def test_the_64_bit_counters_and_high_speed_win(): eth0 = _collect(tables)["interfaces"]["eth0"] - assert (eth0["rx_bytes"], eth0["tx_bytes"], eth0["speed_mbps"]) == (8589934592, 17179869184, 10000) + assert (eth0["rx_bytes"], eth0["tx_bytes"], eth0["speed_mbps"]) == ( + 8589934592, + 17179869184, + 10000, + ) def test_an_agent_without_ifxtable_falls_back_to_the_32_bit_counters(): diff --git a/tests/test_virtual_ips.py b/tests/test_virtual_ips.py index 8ec3cac..da54d55 100644 --- a/tests/test_virtual_ips.py +++ b/tests/test_virtual_ips.py @@ -152,9 +152,7 @@ class TestVipManager: assert parse_virtual_ips(_wire(section))["vip_manager"] == [] def test_a_template_unit_is_named_after_its_instance(self): - section = _vip_manager( - "active active", "cfg ip 10.0.0.10", unit="vip-manager@pg16.service" - ) + section = _vip_manager("active active", "cfg ip 10.0.0.10", unit="vip-manager@pg16.service") [entry] = parse_virtual_ips(_wire(section))["vip_manager"] @@ -496,8 +494,10 @@ class TestTheCommandOnAHost: assert SECRET not in out assert "MASTER" not in out and "priority" not in out reading = parse_virtual_ips(out) - assert [(e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) - for e in reading["keepalived"]] == [ + assert [ + (e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) + for e in reading["keepalived"] + ] == [ ("VI_WEB", "192.0.2.10", 24, "eth0", "vrid:51"), ("VI_DB", "192.0.2.20", 32, "eth1", "vrid:52"), ("VI_DB", "2001:db8::20", 64, "eth1", "vrid:52"), @@ -593,8 +593,10 @@ class TestTheCommandOnAHost: out = host.run() assert SECRET not in out - assert [(e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) - for e in parse_virtual_ips(out)["keepalived"]] == [ + assert [ + (e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) + for e in parse_virtual_ips(out)["keepalived"] + ] == [ ("VI_1", "192.0.2.50", 32, "eth0", None), ("VI_2", "192.0.2.60", 24, "eth1", "vrid:53"), ] diff --git a/tests/test_vm_cpu_types.py b/tests/test_vm_cpu_types.py index 10a2474..14ffd39 100644 --- a/tests/test_vm_cpu_types.py +++ b/tests/test_vm_cpu_types.py @@ -15,7 +15,7 @@ from __future__ import annotations import ast import inspect -from typing import List, get_type_hints +from typing import get_type_hints import napalm_device_types.hypervisor as hypervisor_module from napalm_device_types import HypervisorDriver @@ -52,7 +52,7 @@ class TestCpuTypeListing: assert get_type_hints(VMCpuTypeDict) == { "name": str, "description": str, - "features": List[str], + "features": list[str], "available": bool, "default": bool, } diff --git a/tests/test_vm_identifiers.py b/tests/test_vm_identifiers.py index d38f6e2..3c7752c 100644 --- a/tests/test_vm_identifiers.py +++ b/tests/test_vm_identifiers.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import get_type_hints import pytest + from napalm_device_types.models import ( VMConfigDict, VMDict, From b7350a9b2c2ae177791d9ef0c3fa6982c40186b1 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 22 Oct 2026 19:05:11 +0200 Subject: [PATCH 18/50] fix: what bandit and the remaining lint rules found - Long docstring and command lines are wrapped; recorded device output in tests/ keeps its lines (E501 is off there). - host_firewall: the negation token is the named constant _NEGATION. bandit read token == "!" as a hardcoded password. - ssh_host_keys: _known_hosts_file takes the expected key as an argument instead of asserting it; both callers had already returned when it is None. - ssh_server: the staged path is on the device, not a local temp file (nosec B108). - tests: raise ... from exc, zip(strict=True). --- napalm_device_types/access_point.py | 3 ++- napalm_device_types/host_firewall.py | 16 +++++++++++----- napalm_device_types/hypervisor.py | 3 ++- napalm_device_types/kernel.py | 3 ++- napalm_device_types/mac_acl.py | 12 ++++++++---- napalm_device_types/models.py | 4 +++- napalm_device_types/ssh_host_keys.py | 11 +++++------ napalm_device_types/ssh_server.py | 6 ++++-- napalm_device_types/storage.py | 15 ++++++++++----- pyproject.toml | 4 ++++ tests/test_ssh_host_keys.py | 2 +- tests/test_vm_cpu_types.py | 5 ++++- 12 files changed, 56 insertions(+), 28 deletions(-) diff --git a/napalm_device_types/access_point.py b/napalm_device_types/access_point.py index 94a1146..2e8a9d6 100644 --- a/napalm_device_types/access_point.py +++ b/napalm_device_types/access_point.py @@ -190,7 +190,8 @@ class AccessPointDriver( Keys are SSID names. Each value contains: * ssid (string) - SSID name (repeated for convenience) - * bridge (string) - bridge interface the VAP is attached to (e.g. ``"br-lan"``, ``"br-guest"``) + * bridge (string) - bridge interface the VAP is attached to + (e.g. ``"br-lan"``, ``"br-guest"``) * vlan_id (int) - 802.1Q VLAN ID (0 = untagged / no VLAN separation) * tagged (bool) - whether traffic is 802.1Q-tagged on the uplink port * client_isolation (bool) - whether clients on this SSID are isolated from each other diff --git a/napalm_device_types/host_firewall.py b/napalm_device_types/host_firewall.py index 259f94b..3b03deb 100644 --- a/napalm_device_types/host_firewall.py +++ b/napalm_device_types/host_firewall.py @@ -166,6 +166,9 @@ def _sections(lines: list[str]) -> dict[str, tuple[list[str], int | None]]: # ── iptables-save ──────────────────────────────────────────────────────────── +#: The token that negates the option after it (``! -s 10.0.0.0/8``). +_NEGATION = "!" + #: Options that start a new part of a rule, outside any ``-m`` module. _TOP_LEVEL = frozenset( { @@ -244,13 +247,13 @@ def _module_options(module: str, rt: _RuleTokens, matches: list[NetfilterMatchDi unknown = module not in _KNOWN_MODULES while rt.more(): token = rt.peek() - if token in _TOP_LEVEL or (token == "!" and rt.peek(1) in _TOP_LEVEL): + if token in _TOP_LEVEL or (token == _NEGATION and rt.peek(1) in _TOP_LEVEL): break negated = False - if token == "!": + if token == _NEGATION: rt.take() negated = True - text.append("!") + text.append(_NEGATION) option = rt.take() value = rt.take() if rt.more() and not str(rt.peek()).startswith("-") else "" text += [option] + ([value] if value else []) @@ -266,7 +269,10 @@ def _module_options(module: str, rt: _RuleTokens, matches: list[NetfilterMatchDi def _module_match(module: str, option: str, value: str, negated: bool) -> Any: - """The match one module option makes; False for one that matches nothing new; None if not read.""" + """The match one module option makes. + + False for one that matches nothing new; None if not read. + """ if module == "comment": return False if module in ("tcp", "udp", "sctp", "udplite"): @@ -323,7 +329,7 @@ def _iptables_rule(line: str, chains: set[str]) -> tuple[str, NetfilterRuleDict] } while rt.more(): token = rt.take() - if token == "!": + if token == _NEGATION: negated = True continue if token in simple: diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 8cec25b..5406f1d 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -330,7 +330,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri :param name: VM name or its ``vmid``. :param snapshot: Name of the snapshot to delete. :raises ValueError: If the VM or snapshot does not exist. - :raises RuntimeError: If other snapshots depend on this one (must delete children first). + :raises RuntimeError: If other snapshots depend on this one + (must delete children first). Example:: diff --git a/napalm_device_types/kernel.py b/napalm_device_types/kernel.py index bf31cca..629ddab 100644 --- a/napalm_device_types/kernel.py +++ b/napalm_device_types/kernel.py @@ -41,7 +41,8 @@ KERNEL_FACTS_COMMAND = ( "if [ -r $m/modules.builtin ]; then echo '[builtin]'; cat $m/modules.builtin; fi; " "if [ -r $m/modules.dep ]; then echo '[available]'; cut -d: -f1 $m/modules.dep; fi; " "if [ -r /boot/config-$r ]; then echo '[config]'; grep '^CONFIG_' /boot/config-$r; " - "elif [ -r /proc/config.gz ]; then echo '[config]'; zcat /proc/config.gz | grep '^CONFIG_'; fi; " + "elif [ -r /proc/config.gz ]; then echo '[config]'; " + "zcat /proc/config.gz | grep '^CONFIG_'; fi; " "} 2>/dev/null | gzip -c | base64; " "printf '%s%s\\n' KFACTS_ END" ) diff --git a/napalm_device_types/mac_acl.py b/napalm_device_types/mac_acl.py index a72e5d5..b1d20f9 100644 --- a/napalm_device_types/mac_acl.py +++ b/napalm_device_types/mac_acl.py @@ -43,15 +43,18 @@ class MacAclMixin: "name": "CorpWiFi", "policy": "allow", "entries": [ - {"mac": "AA:BB:CC:DD:EE:01", "action": "allow", "description": "CEO-Laptop"}, - {"mac": "AA:BB:CC:DD:EE:02", "action": "allow", "description": "CFO-Laptop"}, + {"mac": "AA:BB:CC:DD:EE:01", "action": "allow", + "description": "CEO-Laptop"}, + {"mac": "AA:BB:CC:DD:EE:02", "action": "allow", + "description": "CFO-Laptop"}, ], }, "GuestNet": { "name": "GuestNet", "policy": "deny", "entries": [ - {"mac": "DE:AD:BE:EF:00:01", "action": "deny", "description": "blocked device"}, + {"mac": "DE:AD:BE:EF:00:01", "action": "deny", + "description": "blocked device"}, ], }, } @@ -72,7 +75,8 @@ class MacAclMixin: Example:: - driver.push_mac_acl("CorpWiFi", "whitelist", ["AA:BB:CC:DD:EE:01", "AA:BB:CC:DD:EE:02"]) + macs = ["AA:BB:CC:DD:EE:01", "AA:BB:CC:DD:EE:02"] + driver.push_mac_acl("CorpWiFi", "whitelist", macs) driver.push_mac_acl("GuestNet", "off", []) """ ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index e2ae9d0..6b4effc 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1493,7 +1493,9 @@ class ManagementProtocolsDict(TypedDict): class SystemIdentityDict(TypedDict): - """A host's SMBIOS identity, what ties it to its BMC (:mod:`napalm_device_types.system_identity`). + """A host's SMBIOS identity, what ties it to its BMC. + + See :mod:`napalm_device_types.system_identity`. Each value is empty when unknown or a vendor placeholder.""" diff --git a/napalm_device_types/ssh_host_keys.py b/napalm_device_types/ssh_host_keys.py index c1ee214..a9d2df4 100644 --- a/napalm_device_types/ssh_host_keys.py +++ b/napalm_device_types/ssh_host_keys.py @@ -212,7 +212,7 @@ class HostKeyCheck: if self.expected is None: client.set_missing_host_key_policy(_AcceptFirst()) return - with self._known_hosts_file() as path: + with self._known_hosts_file(self.expected) as path: client.get_host_keys().load(path) client.set_missing_host_key_policy(_RefuseUnknown()) @@ -243,7 +243,7 @@ class HostKeyCheck: with self.refusing(): yield kwargs return - with self._known_hosts_file() as path: + with self._known_hosts_file(self.expected) as path: kwargs.update(ssh_strict=True, alt_host_keys=True, alt_key_file=path) with self.refusing(): yield kwargs @@ -285,13 +285,12 @@ class HostKeyCheck: ) @contextmanager - def _known_hosts_file(self) -> Iterator[str]: - """A private ``known_hosts`` file holding only the expected key.""" - assert self.expected is not None + def _known_hosts_file(self, expected: SshHostKey) -> Iterator[str]: + """A private ``known_hosts`` file holding only *expected*.""" fd, path = tempfile.mkstemp(prefix="known_hosts-") try: with os.fdopen(fd, "w") as f: - f.write(self.expected.known_hosts_line(self.host, self.port) + "\n") + f.write(expected.known_hosts_line(self.host, self.port) + "\n") yield path finally: try: diff --git a/napalm_device_types/ssh_server.py b/napalm_device_types/ssh_server.py index e8e33f7..a20c9db 100644 --- a/napalm_device_types/ssh_server.py +++ b/napalm_device_types/ssh_server.py @@ -369,7 +369,8 @@ class OpenSshServerMixin: stdin=content.encode(), ) tmp = staged.stdout.strip() - if staged.exit_code != 0 or not tmp.startswith("/tmp/netork-ssh."): + # A path on the device, made by mktemp under umask 077: not a local temp file. + if staged.exit_code != 0 or not tmp.startswith("/tmp/netork-ssh."): # nosec B108 raise RuntimeError( f"could not stage {path}: {(staged.stderr or staged.stdout).strip()}" ) @@ -378,7 +379,8 @@ class OpenSshServerMixin: # GNU stat, then BSD's; 644 for a file that does not exist yet. f'r={quote(mode_of or path)}; m=$(stat -c %a -- "$r" 2>/dev/null' f' || stat -f %Lp -- "$r" 2>/dev/null) || m=644;' - f' install -m "$m" -- {quote(tmp)} {quote(part)} && mv -f -- {quote(part)} {quote(path)};' + f' install -m "$m" -- {quote(tmp)} {quote(part)}' + f" && mv -f -- {quote(part)} {quote(path)};" f" s=$?; rm -f -- {quote(tmp)} {quote(part)}; exit $s" ) if result.exit_code != 0: diff --git a/napalm_device_types/storage.py b/napalm_device_types/storage.py index 1455221..74c62c4 100644 --- a/napalm_device_types/storage.py +++ b/napalm_device_types/storage.py @@ -63,7 +63,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * rpm (int) - rotational speed; ``0`` for SSD/NVMe * temperature (int) - current temperature in Celsius; ``-1`` if unavailable * health (string) - ``"healthy"``, ``"warning"``, ``"failed"``, or ``"unknown"`` - * pool (string) - name of the pool this disk belongs to; empty string if unassigned/spare + * pool (string) - name of the pool this disk belongs to; + empty string if unassigned/spare Example:: @@ -107,12 +108,14 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * type (string) - ``"zfs"``, ``"lvm"``, ``"md"``, ``"hardware-raid"``, ``"btrfs"`` * level (string) - RAID/redundancy level: ``"mirror"``, ``"raidz1"``, ``"raidz2"``, ``"raidz3"``, ``"stripe"``, ``"raid0"`` … ``"raid60"``, ``"single"`` - * status (string) - ``"online"``, ``"degraded"``, ``"faulted"``, ``"offline"``, ``"unknown"`` + * status (string) - ``"online"``, ``"degraded"``, ``"faulted"``, + ``"offline"``, ``"unknown"`` * total (int) - usable capacity in bytes * used (int) - used space in bytes * available (int) - free space in bytes * disks (list of strings) - slot identifiers of member drives - * auto_expand (bool) - whether the pool grows automatically when disks are replaced with larger ones + * auto_expand (bool) - whether the pool grows automatically when disks + are replaced with larger ones * dedup (bool) - whether deduplication is enabled * compression (string) - pool-level compression algorithm: ``"off"``, ``"lz4"``, ``"gzip"``, ``"zstd"``, etc. @@ -155,7 +158,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * used (int) - space currently used in bytes * available (int) - space available in bytes * mountpoint (string) - local mount path; empty string for block volumes / LUNs - * compression (string) - active compression algorithm (``"off"``, ``"lz4"``, ``"zstd"``, etc.) + * compression (string) - active compression algorithm + (``"off"``, ``"lz4"``, ``"zstd"``, etc.) * dedup (bool) - whether deduplication is active on this volume * readonly (bool) - whether the volume is mounted read-only * snapshots (int) - number of snapshots currently held @@ -254,7 +258,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): Each entry contains: - * name (string) - snapshot name (e.g. ``"auto-2026-05-11"`` or full ``"tank/media@auto-2026-05-11"``) + * name (string) - snapshot name (e.g. ``"auto-2026-05-11"`` + or full ``"tank/media@auto-2026-05-11"``) * volume (string) - parent volume / dataset path * created (float) - creation timestamp (Unix epoch) * size (int) - bytes of unique data referenced only by this snapshot diff --git a/pyproject.toml b/pyproject.toml index 6de333c..e6e83da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,10 @@ target-version = "py310" [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] +[tool.ruff.lint.per-file-ignores] +# Recorded device output keeps its lines. +"tests/**" = ["E501"] + [tool.mypy] python_version = "3.10" strict = true diff --git a/tests/test_ssh_host_keys.py b/tests/test_ssh_host_keys.py index f2d91a8..6470349 100644 --- a/tests/test_ssh_host_keys.py +++ b/tests/test_ssh_host_keys.py @@ -252,7 +252,7 @@ class TestNetmiko: other = paramiko.ECDSAKey.generate() raise paramiko.BadHostKeyException("10.0.0.5", host_key, other) except paramiko.SSHException as exc: - raise RuntimeError(f"A paramiko SSHException occurred: {exc}") + raise RuntimeError(f"A paramiko SSHException occurred: {exc}") from exc except RuntimeError as exc: raise ConnectionException(str(exc)) from exc assert caught.value.observation.seen == SshHostKey.of(host_key) diff --git a/tests/test_vm_cpu_types.py b/tests/test_vm_cpu_types.py index 14ffd39..61b8516 100644 --- a/tests/test_vm_cpu_types.py +++ b/tests/test_vm_cpu_types.py @@ -33,7 +33,10 @@ def _declared(name: str) -> ast.FunctionDef: class TestCreateVmTakesACpuType: def test_cpu_type_is_an_optional_keyword(self): fn = _declared("create_vm_from_cloud_init") - kwonly = {arg.arg: default for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults)} + kwonly = { + arg.arg: default + for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults, strict=True) + } assert "cpu_type" in kwonly default = kwonly["cpu_type"] assert isinstance(default, ast.Constant) and default.value is None From 04751846847caea6e33c6ff66547020204e7e7a7 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 22 Oct 2026 19:06:19 +0200 Subject: [PATCH 19/50] chore: adopt the NAPALM driver standard - CI moves to .gitea/workflows/ci.yml and calls the shared driver workflow of NAPALM/driver-standard: conformance, ruff, strict mypy, tests on 3.10-3.13 and at the lowest declared versions, bandit, pip-audit, build, and on a release tag the upload to PyPI. - tests/quality_baseline.json records today's long and complex functions and the one file over 1000 lines; they may shrink, not grow. - Coverage floor 95 % (measured 95.15 %). - CHANGELOG.md starts here; project links point at git.netork.io; the license is the SPDX string PEP 639 wants. --- .gitea/workflows/ci.yml | 15 ++++++++++++ .github/workflows/ci.yml | 49 ------------------------------------- CHANGELOG.md | 17 +++++++++++++ README.md | 15 ++++++++---- pyproject.toml | 28 +++++++++++++-------- tests/quality_baseline.json | 39 +++++++++++++++++++++++++++++ 6 files changed, 99 insertions(+), 64 deletions(-) create mode 100644 .gitea/workflows/ci.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 CHANGELOG.md create mode 100644 tests/quality_baseline.json diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..7530c79 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,15 @@ +name: CI + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + +jobs: + driver: + uses: NAPALM/driver-standard/.gitea/workflows/driver.yml@main + with: + standard-ref: main + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 372e184..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: CI - -on: - push: - branches: ["**"] - pull_request: - branches: ["**"] - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - - name: Install package with dev extras - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - - name: Type-check - run: | - python -m mypy napalm_device_types - - - name: Run unit tests - run: | - python -m pytest -q --tb=short - - - name: Build wheel and sdist - run: | - python -m pip install build - python -m build - - - name: Upload dist artifacts - # v4 refuses to run on Gitea ("not currently supported on GHES"). - uses: actions/upload-artifact@v3 - with: - name: dist-${{ matrix.python-version }} - path: dist/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..4ccc0f8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to napalm-device-types. The format follows +[Keep a Changelog](https://keepachangelog.com/), versions follow +[Semantic Versioning](https://semver.org/). Versions up to 4.21.0 were not +recorded here; their history is the git log. + +## [Unreleased] + +### Changed +- The package follows the NAPALM driver standard + (https://git.netork.io/NAPALM/driver-standard): CI on Gitea runs ruff, strict + mypy, the tests on Python 3.10–3.13 and at the lowest declared versions, + bandit, pip-audit and a build check, and a release tag publishes to PyPI. +- Type annotations use builtin generics (`list[str]`, `X | None`) instead of + `typing.List` and `Optional`. They mean the same to a type checker. +- The project links point at git.netork.io instead of GitHub. diff --git a/README.md b/README.md index 1689662..b675bbf 100644 --- a/README.md +++ b/README.md @@ -569,7 +569,7 @@ from napalm_device_types.models import ( ## Development ```bash -git clone https://github.com/chrismanivong/napalm-device-types.git +git clone https://git.netork.io/NAPALM/napalm-device-types.git cd napalm-device-types pip install -e ".[dev]" ``` @@ -582,10 +582,15 @@ mypy napalm_device_types ## Contributing -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/router-driver`) -3. Implement your changes -4. Open a Pull Request +This package follows the +[NAPALM driver standard](https://git.netork.io/NAPALM/driver-standard/src/branch/main/STANDARDS.md): +test first, ruff and strict mypy, Conventional Commits, one pull request per change +against `main`. CI checks what a tool can check: + +```bash +pip install "napalm-standard @ git+https://git.netork.io/NAPALM/driver-standard.git" +napalm-standard check && napalm-standard ratchet +``` ## License diff --git a/pyproject.toml b/pyproject.toml index e6e83da..fb84677 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +8,7 @@ version = "4.21.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" -license = { text = "Apache-2.0" } +license = "Apache-2.0" authors = [ { name = "Christian Manivong", email = "christian@manivong.de" }, ] @@ -28,7 +28,6 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Intended Audience :: System Administrators", - "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", @@ -45,16 +44,16 @@ dependencies = [ ] [project.urls] -Homepage = "https://github.com/chrismanivong/napalm-device-types" -Repository = "https://github.com/chrismanivong/napalm-device-types" -Issues = "https://github.com/chrismanivong/napalm-device-types/issues" +Repository = "https://git.netork.io/NAPALM/napalm-device-types" +Issues = "https://git.netork.io/NAPALM/napalm-device-types/issues" +Changelog = "https://git.netork.io/NAPALM/napalm-device-types/src/branch/main/CHANGELOG.md" [project.optional-dependencies] dev = [ - "pytest", - "mypy", - "build", - "twine", + "pytest>=8", + "pytest-cov>=5", + "mypy>=1.11", + "ruff>=0.6", ] [tool.setuptools.packages.find] @@ -79,3 +78,12 @@ select = ["E", "F", "I", "UP", "B"] python_version = "3.10" strict = true ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +source = ["napalm_device_types"] + +[tool.coverage.report] +fail_under = 95 diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json new file mode 100644 index 0000000..488f4d5 --- /dev/null +++ b/tests/quality_baseline.json @@ -0,0 +1,39 @@ +{ + "coverage_floor": 95.0, + "function_complexity": { + "napalm_device_types/_ucd_metrics.py::build_if_metrics": 20, + "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 17, + "napalm_device_types/host_firewall.py::_iptables_rule": 11, + "napalm_device_types/host_firewall.py::_module_match": 13, + "napalm_device_types/host_firewall.py::_module_options": 14, + "napalm_device_types/host_firewall.py::_nft_field": 15, + "napalm_device_types/host_firewall.py::_nft_match": 12, + "napalm_device_types/host_firewall.py::_nft_rule": 16, + "napalm_device_types/host_firewall.py::_parse_nft": 14, + "napalm_device_types/host_firewall.py::rule_text": 14, + "napalm_device_types/host_status.py::_reboot": 12, + "napalm_device_types/lag.py::add_lag_interfaces": 11, + "napalm_device_types/listening.py::parse_listening_sockets": 13, + "napalm_device_types/service_clusters.py::_patroni": 11, + "napalm_device_types/ssh_server.py::plan_ssh_hardening": 38, + "napalm_device_types/virtual_ips.py::_keepalived": 12 + }, + "function_lines": { + "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 119, + "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_reservations": 54, + "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_subnets": 60, + "napalm_device_types/firewall_rules.py::FirewallRuleMixin.diff_firewall_rules": 56, + "napalm_device_types/hypervisor.py::HypervisorDriver.create_vm_from_cloud_init": 84, + "napalm_device_types/hypervisor.py::HypervisorDriver.get_vm_config": 77, + "napalm_device_types/ssh_server.py::OpenSshServerMixin.harden_ssh_server": 52, + "napalm_device_types/ssh_server.py::plan_ssh_hardening": 60, + "napalm_device_types/storage.py::StorageDriver.get_volumes": 52, + "napalm_device_types/switch.py::SwitchDriver.get_dot1x_ports": 51, + "napalm_device_types/switch.py::SwitchDriver.get_poe_status": 63 + }, + "mypy_relaxed": [], + "python_file_lines": { + "napalm_device_types/models.py": 1504 + }, + "untyped_public": [] +} From d7ec5589332d92c38bd81a7d37365fa913b12187 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 22 Oct 2026 19:16:40 +0200 Subject: [PATCH 20/50] feat: AptProxyMixin, what apt fetches through and pointing it at a proxy (NetOrk/netork#1397) apt reads apt.conf.d in order and apt.conf last; the last value wins. The Debian installer writes its proxy into apt.conf, Ubuntu's into 90curtin-aptproxy: reading 00proxy alone missed both, and a fix that wrote 00proxy and read it back reported success while apt fetched elsewhere. - get_package_proxy_report(): apt-config dump's proxy, per-host overrides, auto-detect, every line of apt's files that sets it (in reading order, editable or inside a block), the http and https sources. - check_package_proxy(url, expect): a fetch with apt-helper, past any proxy; an HTTP error counts as an answer. - configure_package_proxy(url): write 00proxy, comment out other proxy lines, read apt's proxy back, fetch one InRelease through it. - same_proxy_url(): scheme, host and port, a trailing slash aside. A driver supplies _run_apt_command, _run_apt_root_command and _write_apt_file. Version 4.22.0. --- README.md | 9 + napalm_device_types/__init__.py | 12 + napalm_device_types/apt_proxy.py | 404 ++++++++++++++++++++++++++ napalm_device_types/models.py | 39 +++ pyproject.toml | 2 +- tests/test_apt_proxy.py | 475 +++++++++++++++++++++++++++++++ 6 files changed, 940 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/apt_proxy.py create mode 100644 tests/test_apt_proxy.py diff --git a/README.md b/README.md index e7c6e8f..7654c37 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,15 @@ that binary links against. An application needs it to let a statement about a se (the postgresql unit) reach the findings on its packages. A driver supplies `_run_service_packages_command`. +`AptProxyMixin` (`napalm_device_types.apt_proxy`, since 4.22) is `PackageProxyMixin` for a +Debian-family host, mixed in the same way. What apt fetches through is what `apt-config dump` +says, not one file: apt reads `apt.conf.d` in order and `apt.conf` last, and the last value +wins (`get_package_proxy`, `get_package_proxy_report` with every line that sets it). +`check_package_proxy(url, expect)` fetches a page with apt's own downloader, past any proxy. +`configure_package_proxy(url)` writes `00proxy`, comments out every other line that sets a +different proxy, reads the result back and fetches one http source's `InRelease` through it. +A driver supplies `_run_apt_command`, `_run_apt_root_command` and `_write_apt_file`. + `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer kernel of the running flavour installed), whether it patches itself (unattended-upgrades, diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 3c8028a..d208dd3 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -153,6 +153,13 @@ from napalm_device_types.phone import PhoneDriver from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status +from napalm_device_types.apt_proxy import ( + APT_PROXY_COMMAND, + AptProxyMixin, + parse_apt_fetch, + parse_apt_proxy, + same_proxy_url, +) from napalm_device_types.package_updates import ( APT_UPGRADABLE_COMMAND, DNF_SECURITY_COMMAND, @@ -221,6 +228,11 @@ __all__ = [ "FixStepDict", "FixStepOutcomeDict", "PackageProxyMixin", + "APT_PROXY_COMMAND", + "AptProxyMixin", + "parse_apt_fetch", + "parse_apt_proxy", + "same_proxy_url", "SnmpAgentMixin", "SnmpAgentSettingsDict", "run_fix_steps", diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py new file mode 100644 index 0000000..7e4530c --- /dev/null +++ b/napalm_device_types/apt_proxy.py @@ -0,0 +1,404 @@ +# -*- coding: utf-8 -*- +"""apt's proxy: what apt fetches through, whether a proxy answers, and pointing apt at one. + +Read the same way on every Debian-family host, so the commands and their parse +live here once, like :mod:`~napalm_device_types.host_status`; a driver only +carries a command across and runs one as root. + +**What apt fetches through** is what ``apt-config dump`` says, not one file. apt +reads every file in ``Dir::Etc::parts`` (``apt.conf.d``) in byte order, then +``Dir::Etc::main`` (``apt.conf``); for a single value the last one read wins. The +Debian installer writes its proxy into ``apt.conf``, Ubuntu's into +``apt.conf.d/90curtin-aptproxy``: both win over an ``00proxy`` +(NetOrk/netork#1397). ``DIRECT`` for the proxy, or for one host +(``Acquire::http::Proxy::``), means no proxy there. + +**Where it is set** is every line of apt's files that sets the http proxy, in the +order apt reads them. A line of the form ``Acquire::http::Proxy "…";`` can be +commented out; a setting inside a block (``Acquire::http { Proxy "…"; };``) is +named but left alone. apt skips a file in ``apt.conf.d`` whose name has other +characters than letters, digits, ``_``, ``-`` and ``.``, or an extension other +than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); so is it here. + +**Whether a proxy answers** is a fetch with apt's own downloader, +``apt-helper download-file``, which is on every apt host and needs no root: +straight to a page the caller names (past any proxy), or through the proxy apt is +set to. An HTTP error is an answer too: the other end was reached. Whether the +page is the right one, the caller says (``expect``). + +**Pointing apt at a proxy** writes :attr:`AptProxyMixin.APT_PROXY_FILE`, comments +out every other line that sets a different one, reads the result back from +``apt-config``, and fetches the ``InRelease`` of one http source through it. The +last step is what proves it: apt reached a mirror through the proxy it is now set +to. The source chosen is one apt does not fetch ``DIRECT``. +""" + +from __future__ import annotations + +import re +import shlex +from typing import Dict, List, Optional, TYPE_CHECKING +from urllib.parse import urlsplit + +from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps +from napalm_device_types.models import ( + AptProxyReportDict, + AptProxySettingDict, + FixResultDict, + FixStepOutcomeDict, + ProxyFetchDict, +) +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "APROXY_BEGIN" +_END = "APROXY_END" +_APT_HELPER = "/usr/lib/apt/apt-helper" + +#: A proxy URL apt's file and a shell word can both carry. +PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") + +#: One line, POSIX ``sh``, read-only, no privileges. In a subshell, so neither the +#: locale nor its variables stay behind in an interactive session; the frame +#: markers are printed in two halves so that an echoing transport does not show +#: them early. +APT_PROXY_COMMAND = ( + "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " + "echo '[dump]'; apt-config dump 2>/dev/null | grep -i '^Acquire::http::Proxy'; " + "eval \"$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)\"; " + "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " + "[ -f \"$f\" ] && grep -Hin proxy \"$f\"; done; " + "echo '[main]'; f=\"${ap_main:-/etc/apt/apt.conf}\"; [ -f \"$f\" ] && grep -Hin proxy \"$f\"; " + "echo '[sources]'; apt-get indextargets --no-release-info --format '$(BASE_URI)' " + "2>/dev/null | sort -u; " + "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" +) + +_DUMP = re.compile(r'^Acquire::http::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) +_GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") +_FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) +_NOT_HTTP_PROXY = re.compile( + r"^\s*(//|#)|Acquire::(https|ftp|socks)\b|::Proxy::|Proxy-Auto-Detect", re.I +) +_IN_BLOCK = re.compile(r'\bProxy\s+"([^"]*)"', re.I) +#: The names apt reads in ``apt.conf.d``: no extension, or ``.conf``. +_PART_NAME = re.compile(r"^(?:[A-Za-z0-9_\-]+|[A-Za-z0-9_.\-]+\.conf)$") +_NO_PROXY = frozenset({"", "direct"}) + + +def _sections(output: str) -> Dict[str, List[str]]: + lines = [line.rstrip() for line in strip_terminal_codes(output).splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact apt proxy report in the output") from None + sections: Dict[str, List[str]] = {} + current: List[str] = [] + for line, bare in zip(lines[start + 1 : end], stripped[start + 1 : end]): + if bare.startswith("[") and bare.endswith("]") and " " not in bare: + current = sections.setdefault(bare[1:-1], []) + elif bare: + current.append(line) + return sections + + +def _setting(line: str, main: bool) -> Optional[AptProxySettingDict]: + found = _GREP.match(line) + if not found: + return None + path, number, text = found.groups() + if not main and not _PART_NAME.match(path.rsplit("/", 1)[-1]): + return None + flat = _FLAT.match(text) + if flat: + return {"file": path, "line": int(number), "value": flat.group(1), "editable": True} + if _NOT_HTTP_PROXY.search(text): + return None + nested = _IN_BLOCK.search(text) + if not nested: + return None + return {"file": path, "line": int(number), "value": nested.group(1), "editable": False} + + +def parse_apt_proxy(output: str) -> AptProxyReportDict: + """The report :data:`APT_PROXY_COMMAND` prints, as :class:`AptProxyReportDict`. + + :raises ValueError: if the output carried no intact report. + """ + sections = _sections(output) + proxy: Optional[str] = None + per_host: Dict[str, str] = {} + auto_detect: Optional[str] = None + for line in sections.get("dump", []): + found = _DUMP.match(line.strip()) + if not found: + continue + suffix, value = found.groups() + if not suffix: + proxy = None if value.lower() in _NO_PROXY else value + elif suffix.lower() == "-auto-detect": + auto_detect = value or None + else: + per_host[suffix[2:]] = value + settings = [ + s + for main, key in ((False, "parts"), (True, "main")) + for s in (_setting(line, main) for line in sections.get(key, [])) + if s + ] + sources = [line.strip() for line in sections.get("sources", [])] + return { + "proxy": proxy, + "per_host": per_host, + "auto_detect": auto_detect, + "settings": settings, + "http_sources": [s for s in sources if s.startswith("http://")], + "https_sources": [s for s in sources if s.startswith("https://")], + } + + +def same_proxy_url(a: Optional[str], b: Optional[str]) -> bool: + """Whether *a* and *b* name the same proxy: scheme, host and port, a trailing ``/`` aside.""" + if not a or not b: + return False + try: + x, y = urlsplit(a), urlsplit(b) + ports = (x.port or _DEFAULT_PORT.get(x.scheme), y.port or _DEFAULT_PORT.get(y.scheme)) + except ValueError: + return False + return ( + x.scheme == y.scheme + and (x.hostname or "") == (y.hostname or "") + and ports[0] == ports[1] + and x.path.rstrip("/") == y.path.rstrip("/") + ) + + +_DEFAULT_PORT = {"http": 80, "https": 443} + + +def apt_fetch_command(url: str, *, direct: bool = False, expect: str = "") -> str: + """The command that fetches *url* with apt's downloader into a temporary file. + + *direct* goes past any proxy apt is set to, to *url*'s host itself; without it + the fetch takes apt's own way. *expect* is text the page should contain. + + :raises ValueError: if *url* is not an http URL a shell word can carry. + """ + if not PROXY_URL.match(url) or not url.startswith("http://"): + raise ValueError(f"not an http URL: {url!r}") + options = ["-o Acquire::Retries=0", "-o Acquire::http::Timeout=10"] + host = urlsplit(url).hostname or "" + if direct and host and ":" not in host: + # *host* comes from a URL PROXY_URL let through: no quote in it. + options.append(f"-o 'Acquire::http::Proxy::{host}=DIRECT'") + match = f"echo '[match]'; grep -c -F -- {shlex.quote(expect)} \"$t\"; " if expect else "" + return ( + "( export LC_ALL=C; t=$(mktemp) || exit 1; printf '%s%s\\n' APROXY_ BEGIN; " + f"echo '[fetch]'; {_APT_HELPER} {' '.join(options)} download-file " + f"{shlex.quote(url)} \"$t\" 2>&1; rc=$?; echo '[rc]'; echo \"$rc\"; " + f"{match}rm -f \"$t\"; printf '%s%s\\n' APROXY_ END ) | cat" + ) + + +_FAILED = re.compile(r"^E: Failed to fetch \S+\s+(.*)$") +_HTTP_STATUS = re.compile(r"^\d{3}\s") + + +def parse_apt_fetch(output: str) -> ProxyFetchDict: + """What :func:`apt_fetch_command` printed, as :class:`ProxyFetchDict`. + + :raises ValueError: if the output carried no intact report, or apt-helper is + missing -- then nothing could be tried. + """ + sections = _sections(output) + try: + rc = int(sections["rc"][0]) + except (KeyError, IndexError, ValueError): + raise ValueError("no exit status in the apt fetch report") from None + if rc in (126, 127): + raise ValueError(f"{_APT_HELPER} cannot run on this host") + match = sections.get("match") + if rc == 0: + expected = (match[0].strip() not in ("", "0")) if match else None + return {"reachable": True, "expected": expected, "error": ""} + error = next( + (m.group(1).strip() for m in map(_FAILED.match, sections.get("fetch", [])) if m), + "the download failed", + ) + answered = bool(_HTTP_STATUS.match(error)) + return {"reachable": answered, "expected": False if answered else None, "error": error} + + +class AptProxyMixin(PackageProxyMixin): + """:class:`~napalm_device_types.fixes.PackageProxyMixin` for a Debian-family host. + + The template form, like :class:`~napalm_device_types.host_status.HostStatusMixin`: + the reading, the parse and the order of the fix are the same everywhere; a + driver supplies the transports below. Mixed in by the drivers that can, so + ``callable(getattr(driver, "configure_package_proxy", None))`` stays a truthful + answer. + """ + + #: The file the proxy is written to. Its name carries no application's. + APT_PROXY_FILE = "/etc/apt/apt.conf.d/00proxy" + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_apt_command(self, command: str) -> str: + """Run *command* on the host with ``sh``, without privileges; what it printed.""" + ... + + def _run_apt_root_command(self, command: str, done: str) -> FixStepOutcomeDict: + """Run *command* as root; ``ok`` with *done* when it exited 0.""" + ... + + def _write_apt_file(self, path: str, content: str) -> FixStepOutcomeDict: + """Replace *path* with *content*, as a new file owned by root, mode 644.""" + ... + + def _is_apt_host(self) -> bool: + """Whether apt is the host's package manager; a driver that can tell overrides it.""" + return True + + def get_package_proxy_report(self) -> Optional[AptProxyReportDict]: + """What apt fetches through and where that is set; None on a host without apt. + + * proxy (string or None): the http proxy apt fetches through + * per_host ({host: proxy or "DIRECT"}) + * auto_detect (string or None): ``Acquire::http::Proxy-Auto-Detect`` + * settings (list of {file, line, value, editable}), in apt's reading order + * http_sources, https_sources (lists of the sources' base URIs) + + :raises ValueError: if the host's output carried no intact report. + """ + if not self._is_apt_host(): + return None + return parse_apt_proxy(self._run_apt_command(APT_PROXY_COMMAND)) + + def get_package_proxy(self) -> Optional[str]: + """The proxy apt fetches through now; None for none, or a host without apt.""" + report = self.get_package_proxy_report() + return report["proxy"] if report else None + + def check_package_proxy(self, url: str, expect: str = "") -> ProxyFetchDict: + """Fetch *url* from the host, past any proxy apt is set to. + + Whether the host reaches it, and whether the page contains *expect*. + + :raises ValueError: if *url* is no http URL, or the host cannot fetch at all. + :raises NotImplementedError: on a host without apt. + """ + if not self._is_apt_host(): + raise NotImplementedError("not an apt host") + return parse_apt_fetch( + self._run_apt_command(apt_fetch_command(url, direct=True, expect=expect)) + ) + + def configure_package_proxy(self, url: str) -> FixResultDict: + """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`.""" + if not url: + return run_fix_steps([("configure", self._remove_package_proxy)]) + return run_fix_steps( + [ + ("configure", lambda: self._write_package_proxy(url)), + ("conflicts", lambda: self._comment_out_other_proxies(url)), + ("verify", lambda: self._verify_package_proxy(url)), + ("fetch", lambda: self._fetch_through_package_proxy(url)), + ] + ) + + def _not_apt(self) -> FixStepOutcomeDict: + return {"ok": False, "output": "not an apt host"} + + def _remove_package_proxy(self) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + return self._run_apt_root_command( + f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" + ) + + def _write_package_proxy(self, url: str) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + if not PROXY_URL.match(url): + raise ValueError(f"not a proxy URL: {url!r}") + return self._write_apt_file(self.APT_PROXY_FILE, f'Acquire::http::Proxy "{url}";\n') + + def _report(self) -> AptProxyReportDict: + report = self.get_package_proxy_report() + if report is None: + raise ValueError("not an apt host") + return report + + def _comment_out_other_proxies(self, url: str) -> FixStepOutcomeDict: + others = [ + s + for s in self._report()["settings"] + if s["file"] != self.APT_PROXY_FILE and not same_proxy_url(s["value"], url) + ] + by_file: Dict[str, List[int]] = {} + for s in others: + if s["editable"]: + by_file.setdefault(s["file"], []).append(s["line"]) + done: List[str] = [] + for path, numbers in by_file.items(): + edits = " ".join( + f"-e '{n}{{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}}'" + for n in numbers + ) + outcome = self._run_apt_root_command( + f"sed -i {edits} {shlex.quote(path)}", f"commented out in {path}" + ) + if not outcome["ok"]: + return outcome + for s in others: + what = "commented out" if s["editable"] else "left alone (inside a block)" + done.append(f"{s['file']}:{s['line']} ({s['value']}) {what}") + return {"ok": True, "output": "; ".join(done) or "no other proxy set"} + + def _verify_package_proxy(self, url: str) -> FixStepOutcomeDict: + report = self._report() + if same_proxy_url(report["proxy"], url): + return {"ok": True, "output": f"apt fetches through {url}"} + where = [ + f"{s['file']}:{s['line']}" + for s in report["settings"] + if s["value"] == report["proxy"] and s["file"] != self.APT_PROXY_FILE + ] + return { + "ok": False, + "output": f"apt still fetches through {report['proxy'] or 'no proxy'}" + + (f", set in {', '.join(where)}" if where else ""), + } + + def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict: + report = self._report() + direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} + source = next( + ( + s + for s in report["http_sources"] + if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct + ), + None, + ) + if source is None: + return {"ok": True, "output": "no http source to fetch through the proxy; not tried"} + target = source.rstrip("/") + "/InRelease" + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if fetched["reachable"] and not fetched["error"]: + return {"ok": True, "output": f"fetched {target} through {url}"} + return {"ok": False, "output": f"{target}: {fetched['error']}"} + + +__all__ = [ + "APT_PROXY_COMMAND", + "AptProxyMixin", + "PROXY_URL", + "apt_fetch_command", + "parse_apt_fetch", + "parse_apt_proxy", + "same_proxy_url", +] diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 9ec1641..90ea84a 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -120,6 +120,45 @@ class HostStatusDict(TypedDict): filesystems: NotRequired[Optional[List[FilesystemUsageDict]]] +class AptProxySettingDict(TypedDict): + """One line of apt's configuration that sets the http proxy (``AptProxyMixin``).""" + + file: str + line: int # 1-based + value: str + #: True for ``Acquire::http::Proxy "…";`` on a line of its own, which can be + #: commented out; False for a setting inside a block. + editable: bool + + +class AptProxyReportDict(TypedDict): + """What apt fetches through, and where that is set (``get_package_proxy_report``).""" + + #: The http proxy apt fetches through; None for none or ``DIRECT``. + proxy: Optional[str] + #: ``Acquire::http::Proxy::``: a proxy, or ``DIRECT``, for one host. + per_host: Dict[str, str] + #: ``Acquire::http::Proxy-Auto-Detect``: a program that names the proxy. + auto_detect: Optional[str] + #: Every line that sets it, in the order apt reads them; the last one wins. + settings: List[AptProxySettingDict] + #: The base URIs of the host's sources over http and over https. + http_sources: List[str] + https_sources: List[str] + + +class ProxyFetchDict(TypedDict): + """A fetch from the host with apt's downloader (``check_package_proxy``).""" + + #: Whether the other end answered at all; an HTTP error is an answer. + reachable: bool + #: Whether the page contained what the caller expected; None when it named nothing + #: or nothing came back. + expected: Optional[bool] + #: apt's reason when the fetch failed, else "". + error: str + + # --------------------------------------------------------------------------- # Access Point # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 8b0471a..be460d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.21.0" +version = "4.22.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py new file mode 100644 index 0000000..8ca4971 --- /dev/null +++ b/tests/test_apt_proxy.py @@ -0,0 +1,475 @@ +"""apt's proxy: what apt fetches through, whether a proxy answers, pointing apt at one. + +apt reads every file in ``apt.conf.d`` in order and ``apt.conf`` last; the last +value wins. The Debian installer writes its proxy into ``apt.conf``, Ubuntu's +into ``apt.conf.d/90curtin-aptproxy``: reading one file was not enough +(NetOrk/netork#1397). The fetch outputs are apt-helper's own, from a Mint 22.3 +host against netOrk's test server. +""" + +from __future__ import annotations + +import os +import re +import subprocess + +import pytest + +from napalm_device_types.apt_proxy import ( + APT_PROXY_COMMAND, + AptProxyMixin, + apt_fetch_command, + parse_apt_fetch, + parse_apt_proxy, + same_proxy_url, +) +from napalm_device_types.fixes import PackageProxyMixin + + +def _report(dump=(), parts=(), main=(), sources=()): + lines = ["APROXY_BEGIN", "[dump]", *dump, "[parts]", *parts, "[main]", *main] + return "\n".join([*lines, "[sources]", *sources, "APROXY_END"]) + + +_D = "/etc/apt/apt.conf.d/" + + +class TestEffectiveProxy: + def test_the_proxy_apt_fetches_through(self): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://c:3142";'], + parts=[f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";'], + ) + ) + + assert report["proxy"] == "http://c:3142" + assert report["settings"] == [ + {"file": f"{_D}00proxy", "line": 1, "value": "http://c:3142", "editable": True} + ] + + def test_none_without_a_proxy(self): + report = parse_apt_proxy(_report()) + + assert report["proxy"] is None + assert report["settings"] == [] + assert report["per_host"] == {} + assert report["auto_detect"] is None + + @pytest.mark.parametrize("value", ["", "DIRECT", "direct"]) + def test_an_empty_or_direct_proxy_is_none(self, value): + assert parse_apt_proxy(_report(dump=[f'Acquire::http::Proxy "{value}";']))["proxy"] is None + + def test_keys_are_case_insensitive(self): + report = parse_apt_proxy(_report(dump=['acquire::HTTP::proxy "http://c:3142";'])) + assert report["proxy"] == "http://c:3142" + + def test_per_host_overrides_and_auto_detect(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::http::Proxy::deb.debian.org "DIRECT";', + 'Acquire::http::Proxy::ppa.launchpad.net "http://other:3128";', + 'Acquire::http::Proxy-Auto-Detect "/usr/bin/auto-apt-proxy";', + ] + ) + ) + + assert report["proxy"] == "http://c:3142" + assert report["per_host"] == { + "deb.debian.org": "DIRECT", + "ppa.launchpad.net": "http://other:3128", + } + assert report["auto_detect"] == "/usr/bin/auto-apt-proxy" + + def test_no_intact_report_raises(self): + with pytest.raises(ValueError): + parse_apt_proxy("APROXY_BEGIN\n[dump]\n") + + +class TestWhereItIsSet: + def test_every_setting_in_apt_s_reading_order(self): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://old:3128";'], + parts=[ + f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";', + f'{_D}90curtin-aptproxy:1:Acquire::http::Proxy "http://old:3128";', + ], + main=['/etc/apt/apt.conf:4: Acquire::http::Proxy "http://older:8080";'], + ) + ) + + assert [(s["file"], s["line"], s["value"]) for s in report["settings"]] == [ + (f"{_D}00proxy", 1, "http://c:3142"), + (f"{_D}90curtin-aptproxy", 1, "http://old:3128"), + ("/etc/apt/apt.conf", 4, "http://older:8080"), + ] + + def test_a_nested_setting_is_named_but_not_editable(self): + report = parse_apt_proxy( + _report(parts=[f'{_D}01proxy:2: Proxy "http://nested:3142";']) + ) + + assert report["settings"] == [ + {"file": f"{_D}01proxy", "line": 2, "value": "http://nested:3142", "editable": False} + ] + + @pytest.mark.parametrize( + "line", + [ + '// Acquire::http::Proxy "http://c:3142";', + '# Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy "http://c:3142";', + 'Acquire::ftp::Proxy "http://c:3142";', + 'Acquire::http::Proxy::deb.debian.org "DIRECT";', + 'Acquire::http::Proxy-Auto-Detect "/usr/bin/x";', + "Acquire::http {", + ], + ) + def test_what_does_not_set_the_http_proxy_is_left_out(self, line): + assert parse_apt_proxy(_report(parts=[f"{_D}00x:1:{line}"]))["settings"] == [] + + @pytest.mark.parametrize("name", ["00proxy.bak", "00proxy.dpkg-old", "00proxy.save", "a b"]) + def test_files_apt_does_not_read_are_left_out(self, name): + line = f'{_D}{name}:1:Acquire::http::Proxy "http://c:3142";' + assert parse_apt_proxy(_report(parts=[line]))["settings"] == [] + + def test_a_conf_extension_is_read(self): + line = f'{_D}01proxy.conf:1:Acquire::http::Proxy "http://c:3142";' + assert len(parse_apt_proxy(_report(parts=[line]))["settings"]) == 1 + + +class TestSources: + def test_https_sources_bypass_an_http_proxy(self): + report = parse_apt_proxy( + _report( + sources=[ + "http://archive.ubuntu.com/ubuntu/dists/noble/", + "https://download.docker.com/linux/ubuntu/dists/noble/", + ] + ) + ) + + assert report["http_sources"] == ["http://archive.ubuntu.com/ubuntu/dists/noble/"] + assert report["https_sources"] == ["https://download.docker.com/linux/ubuntu/dists/noble/"] + + +class TestSameProxyUrl: + @pytest.mark.parametrize( + "a, b", + [ + ("http://c:3142", "http://c:3142/"), + ("http://C:3142", "http://c:3142"), + ("http://c", "http://c:80"), + ], + ) + def test_the_same(self, a, b): + assert same_proxy_url(a, b) + + @pytest.mark.parametrize( + "a, b", + [ + ("http://c:3142", "http://c:31420"), + ("http://c:3142", "http://c:3142.evil"), + ("http://c:3142", "https://c:3142"), + ("http://c:3142", ""), + ("http://c:3142", None), + ], + ) + def test_not_the_same(self, a, b): + assert not same_proxy_url(a, b) + + +_REFUSED = """Err:1 http://172.22.8.50:3143/acng-report.html + Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused) +E: Failed to fetch http://172.22.8.50:3143/acng-report.html Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused) +E: Download Failed""" + +_HTTP_ERROR = """Err:1 http://172.22.8.50:3142/nothing-here.html + 503 Host not found [IP: 172.22.8.50 3142] +E: Failed to fetch http://172.22.8.50:3142/nothing-here.html 503 Host not found [IP: 172.22.8.50 3142] +E: Download Failed""" + +_UNRESOLVED = """Err:1 http://no-such-host.invalid:3142/acng-report.html + Could not resolve 'no-such-host.invalid' +E: Failed to fetch http://no-such-host.invalid:3142/acng-report.html Could not resolve 'no-such-host.invalid' +E: Download Failed""" + +_FETCHED = """Get:1 http://172.22.8.50:3142/acng-report.html +Fetched 14.0 kB in 0s (0 B/s)""" + + +def _fetch(output, rc, match=None): + lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] + if match is not None: + lines += ["[match]", str(match)] + return "\n".join([*lines, "APROXY_END"]) + + +class TestFetch: + def test_fetched_and_expected(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0, match=4)) == { + "reachable": True, + "expected": True, + "error": "", + } + + def test_fetched_but_not_the_expected_page(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0, match=0))["expected"] is False + + def test_without_an_expectation_expected_is_none(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0))["expected"] is None + + def test_refused(self): + result = parse_apt_fetch(_fetch(_REFUSED, 100)) + + assert result["reachable"] is False + assert result["error"] == ( + "Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused)" + ) + + def test_unresolved(self): + result = parse_apt_fetch(_fetch(_UNRESOLVED, 100)) + assert (result["reachable"], result["error"]) == ( + False, + "Could not resolve 'no-such-host.invalid'", + ) + + def test_an_http_error_was_an_answer(self): + result = parse_apt_fetch(_fetch(_HTTP_ERROR, 100)) + + assert result["reachable"] is True + assert result["expected"] is False + assert result["error"] == "503 Host not found [IP: 172.22.8.50 3142]" + + def test_without_apt_helper_nothing_could_be_tried(self): + with pytest.raises(ValueError, match="apt-helper"): + parse_apt_fetch(_fetch("sh: 1: /usr/lib/apt/apt-helper: not found", 127)) + + def test_no_intact_report_raises(self): + with pytest.raises(ValueError): + parse_apt_fetch("APROXY_BEGIN\n[fetch]\n") + + +class TestFetchCommand: + def test_past_the_proxy_to_the_url_itself(self): + command = apt_fetch_command("http://c:3142/acng-report.html", direct=True, expect="Cache NG") + + assert "'Acquire::http::Proxy::c=DIRECT'" in command + assert "grep -c -F -- 'Cache NG'" in command + + def test_through_apt_s_own_proxy(self): + command = apt_fetch_command("http://archive.ubuntu.com/ubuntu/dists/noble/InRelease") + assert "DIRECT" not in command + + @pytest.mark.parametrize("url", ["http://c:3142/x'; rm -rf /", "ftp://c/x", "c:3142"]) + def test_a_url_that_would_break_the_command_is_refused(self, url): + with pytest.raises(ValueError): + apt_fetch_command(url) + + +def test_the_commands_are_valid_posix_sh(): + for command in (APT_PROXY_COMMAND, apt_fetch_command("http://c:3142/x", direct=True, expect="y")): + subprocess.run(["sh", "-n", "-c", command], check=True) + + +@pytest.mark.skipif(not os.path.exists("/usr/bin/apt-config"), reason="needs an apt host") +def test_the_report_command_on_this_host(): + output = subprocess.run(["sh", "-c", APT_PROXY_COMMAND], capture_output=True, text=True).stdout + report = parse_apt_proxy(output) + assert set(report) >= {"proxy", "settings", "https_sources"} + + +class _Host(AptProxyMixin): + """A Debian host: its apt files, and what apt-config would make of them.""" + + def __init__(self, files=None, fetch_rc=0, apt=True): + self.files = dict(files or {}) + self.fetch_rc = fetch_rc + self.apt = apt + self.root = [] + self.fetched = [] + + def _is_apt_host(self): + return self.apt + + def _settings(self): + out = [] + for path in sorted(p for p in self.files if p.startswith(_D)) + [ + p for p in self.files if p == "/etc/apt/apt.conf" + ]: + for n, text in enumerate(self.files[path].splitlines(), 1): + if "proxy" in text.lower(): + out.append((path, n, text)) + return out + + def _effective(self): + value = None + for _path, _n, text in self._settings(): + m = re.match(r'\s*Acquire::http::Proxy\s+"([^"]*)"', text, re.I) + if m: + value = m.group(1) + return value + + def _run_apt_command(self, command): + if command == APT_PROXY_COMMAND: + value = self._effective() + return _report( + dump=[f'Acquire::http::Proxy "{value}";'] if value else [], + parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], + main=[f"{p}:{n}:{t}" for p, n, t in self._settings() if not p.startswith(_D)], + sources=["http://deb.debian.org/debian/dists/trixie/"], + ) + self.fetched.append(command) + return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) + + def _write_apt_file(self, path, content): + self.files[path] = content + return {"ok": True, "output": f"wrote {path}"} + + def _run_apt_root_command(self, command, done): + self.root.append(command) + path = command.rsplit(" ", 1)[-1].strip("'") + for m in re.finditer(r"(\d+)\{/[^}]*\}", command): + n = int(m.group(1)) + lines = self.files[path].splitlines() + lines[n - 1] = "// " + lines[n - 1] + self.files[path] = "\n".join(lines) + "\n" + if command.startswith("rm -f "): + self.files.pop(command.split()[-1].strip("'"), None) + return {"ok": True, "output": done} + + +class TestTheMixin: + def test_it_is_a_package_proxy(self): + assert issubclass(AptProxyMixin, PackageProxyMixin) + + def test_the_fact(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + + assert host.get_package_proxy() == "http://c:3142" + assert host.get_package_proxy_report()["settings"][0]["file"] == f"{_D}00proxy" + + def test_not_an_apt_host(self): + host = _Host(apt=False) + + assert host.get_package_proxy() is None + assert host.get_package_proxy_report() is None + + def test_check_fetches_the_page_past_any_proxy(self): + host = _Host() + + result = host.check_package_proxy("http://c:3142/acng-report.html", expect="Cache") + + assert result == {"reachable": True, "expected": True, "error": ""} + assert "Proxy::c=DIRECT" in host.fetched[0] + + +class TestConfigurePackageProxy: + def test_a_fresh_host(self): + host = _Host() + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert [s["step"] for s in result["steps"]] == ["configure", "conflicts", "verify", "fetch"] + assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' + assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] + + def test_a_later_file_is_commented_out_and_named(self): + host = _Host( + { + f"{_D}90curtin-aptproxy": 'Acquire::http::Proxy "http://old:3128";\n', + "/etc/apt/apt.conf": 'APT::Foo "1";\nAcquire::http::Proxy "http://older:8080";\n', + } + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}90curtin-aptproxy"].startswith("// Acquire::http::Proxy") + assert host.files["/etc/apt/apt.conf"] == ( + 'APT::Foo "1";\n// Acquire::http::Proxy "http://older:8080";\n' + ) + conflicts = result["steps"][1]["output"] + assert f"{_D}90curtin-aptproxy:1 (http://old:3128)" in conflicts + assert "/etc/apt/apt.conf:2 (http://older:8080)" in conflicts + + def test_the_same_proxy_elsewhere_stays(self): + host = _Host({f"{_D}01proxy": 'Acquire::http::Proxy "http://c:3142/";\n'}) + + host.configure_package_proxy("http://c:3142") + + assert host.root == [] + + def test_a_setting_it_cannot_comment_out_fails_verification(self): + host = _Host({"/etc/apt/apt.conf": 'Acquire::http { Proxy "http://older:8080"; };\n'}) + host._effective = lambda: "http://older:8080" + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "verify" + assert "http://older:8080" in result["steps"][-1]["output"] + assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"] + + def test_a_dead_proxy_fails_the_fetch(self): + host = _Host(fetch_rc=100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "fetch" + assert "Connection refused" in result["steps"][-1]["output"] + + def test_without_an_http_source_the_fetch_says_so(self): + host = _Host() + real = host._run_apt_command + + def no_sources(command): + return real(command).replace("http://deb.debian.org/debian/dists/trixie/\n", "") + + host._run_apt_command = no_sources + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "no http source" in result["steps"][-1]["output"] + + def test_a_direct_source_is_not_proof(self): + host = _Host() + real = host._run_apt_command + host._run_apt_command = lambda command: real(command).replace( + "[parts]", 'Acquire::http::Proxy::deb.debian.org "DIRECT";\n[parts]' + ) + + result = host.configure_package_proxy("http://c:3142") + + assert "no http source" in result["steps"][-1]["output"] + + @pytest.mark.parametrize("url", ['http://x"; Acquire::x "y', "ftp://c", "c:3142"]) + def test_a_url_that_would_break_the_file_is_refused(self, url): + host = _Host() + + result = host.configure_package_proxy(url) + + assert result["success"] is False + assert host.files == {} + + def test_an_empty_url_removes_the_file(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + + result = host.configure_package_proxy("") + + assert result["success"] is True + assert host.files == {} + + def test_only_an_apt_host_takes_one(self): + host = _Host(apt=False) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert host.files == {} From 49830671c91fa926b2e93e6ed97333e87bb836e9 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 22 Oct 2026 21:12:28 +0200 Subject: [PATCH 21/50] chore(release): 4.21.1 --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ccc0f8..e590d85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ recorded here; their history is the git log. ## [Unreleased] +## [4.21.1] - 2026-10-22 + +The first release on PyPI since 0.1.0. + ### Changed - The package follows the NAPALM driver standard (https://git.netork.io/NAPALM/driver-standard): CI on Gitea runs ruff, strict diff --git a/pyproject.toml b/pyproject.toml index fb84677..6d850cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.21.0" +version = "4.21.1" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" From 4d3574cd605e6f625413077c2449f53fc2636efa Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 27 Oct 2026 00:36:54 +0100 Subject: [PATCH 22/50] feat: shutdown_host and begin_stop_vm, shutting down without waiting HostShutdownMixin.shutdown_host(), on DeviceTypeDriver: a device shuts itself down cleanly through its own means; declared, so hasattr says whether a driver can. There is no forced variant. VmStopRequestMixin.begin_stop_vm(name): ask for a VM's clean shutdown and return at once, for a caller that waits on a timer and reads get_vm_status instead of blocking in stop_vm. No role demands it. For netOrk's graph workflows, which switch a hypervisor off when CI has nothing for it (NetOrk/netork#1636). --- CHANGELOG.md | 8 +++++++ README.md | 6 +++-- napalm_device_types/__init__.py | 6 +++++ napalm_device_types/base.py | 9 +++++++- napalm_device_types/host_shutdown.py | 32 ++++++++++++++++++++++++++ napalm_device_types/vm_stop_request.py | 29 +++++++++++++++++++++++ tests/test_host_shutdown.py | 27 ++++++++++++++++++++++ tests/test_vm_stop_request.py | 28 ++++++++++++++++++++++ 8 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 napalm_device_types/host_shutdown.py create mode 100644 napalm_device_types/vm_stop_request.py create mode 100644 tests/test_host_shutdown.py create mode 100644 tests/test_vm_stop_request.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b0b69a..94e7a17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ recorded here; their history is the git log. ## [Unreleased] +### Added +- `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself + down cleanly through its own means; declared, so `hasattr` says whether a driver + can. No forced variant (NetOrk/netork#1636). +- `VmStopRequestMixin.begin_stop_vm(name)`: ask for a VM's clean shutdown and return + at once, for callers that wait on a timer instead of in `stop_vm`. No role demands + it (NetOrk/netork#1636). + ## [4.22.0] - 2026-10-23 ### Added diff --git a/README.md b/README.md index ce16fd1..44c28e0 100644 --- a/README.md +++ b/README.md @@ -91,8 +91,10 @@ Behaviour shared across roles lives in a function class, exactly once, and a rol is a thin bundle over them — `PackageManagementMixin`, `HealthMetricsMixin`, `ServiceControlMixin`, `UpdateMixin`, `NatVpnMixin`, `MacAclMixin`, `FirewallRuleMixin`, `DhcpServerMixin`, `PingSweepMixin`, `ConfigLifecycleMixin`, `InterfaceFilterMixin`. -`HostRebootMixin` (`reboot_host`) is mixed into `DeviceTypeDriver` itself, since any -device may be restartable; like the others it only declares. +`HostRebootMixin` (`reboot_host`) and `HostShutdownMixin` (`shutdown_host`) are mixed +into `DeviceTypeDriver` itself, since any device may be restartable or able to shut itself +down; like the others they only declare. `VmStopRequestMixin` (`begin_stop_vm`) asks for a +VM's clean shutdown without waiting for it — a hypervisor driver mixes it in when it can. **Fixes are templates** (`napalm_device_types.fixes`, since 4.13). A change the application wants made on a device runs as fixed steps the driver implements, with values diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index f0d0274..9cc53bf 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -44,6 +44,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.health_metrics.HealthMetricsMixin` * :class:`~napalm_device_types.host_status.HostStatusMixin` * :class:`~napalm_device_types.host_reboot.HostRebootMixin` +* :class:`~napalm_device_types.host_shutdown.HostShutdownMixin` * :class:`~napalm_device_types.interface_filter.InterfaceFilterMixin` * :class:`~napalm_device_types.kernel.KernelFactsMixin` * :class:`~napalm_device_types.listening.ListeningSocketsMixin` @@ -58,6 +59,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.systemd.SystemdServicesMixin` * :class:`~napalm_device_types.updates.UpdateMixin` * :class:`~napalm_device_types.virtual_ips.VirtualIpsMixin` +* :class:`~napalm_device_types.vm_stop_request.VmStopRequestMixin` Introspection -- :func:`~napalm_device_types.roles.roles_of`, :func:`~napalm_device_types.roles.role_keys_of` and @@ -106,6 +108,7 @@ from napalm_device_types.host_firewall import ( parse_host_firewall, ) from napalm_device_types.host_reboot import HostRebootMixin +from napalm_device_types.host_shutdown import HostShutdownMixin from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status from napalm_device_types.hypervisor import HypervisorDriver from napalm_device_types.interface_filter import InterfaceFilterMixin @@ -214,6 +217,7 @@ from napalm_device_types.virtual_ips import ( VirtualIpsMixin, parse_virtual_ips, ) +from napalm_device_types.vm_stop_request import VmStopRequestMixin __all__ = [ "HypervisorHostDict", @@ -269,6 +273,7 @@ __all__ = [ "HOST_STATUS_COMMAND", "HostStatusMixin", "HostRebootMixin", + "HostShutdownMixin", "HypervisorDriver", "InterfaceFilterMixin", "MacAclMixin", @@ -317,6 +322,7 @@ __all__ = [ "UpdateMixin", "VIRTUAL_IPS_COMMAND", "VirtualIpsMixin", + "VmStopRequestMixin", "SERVICE_CLUSTERS_COMMAND", "ServiceClustersMixin", "parse_service_clusters", diff --git a/napalm_device_types/base.py b/napalm_device_types/base.py index 97f0f9b..b95622f 100644 --- a/napalm_device_types/base.py +++ b/napalm_device_types/base.py @@ -15,6 +15,7 @@ from napalm.base import NetworkDriver from napalm_device_types.channel import CommandChannelMixin from napalm_device_types.host_reboot import HostRebootMixin +from napalm_device_types.host_shutdown import HostShutdownMixin from napalm_device_types.ping_sweep import PingSweepMixin @@ -74,7 +75,13 @@ class PortSpec(NamedTuple): mandatory: bool = False -class DeviceTypeDriver(PingSweepMixin, HostRebootMixin, CommandChannelMixin, NetworkDriver): # type: ignore[misc] +class DeviceTypeDriver( + PingSweepMixin, + HostRebootMixin, + HostShutdownMixin, + CommandChannelMixin, + NetworkDriver, # type: ignore[misc] +): """Common base for all netOrk device-type drivers. Sits between napalm.base.NetworkDriver and the type-specific abstract diff --git a/napalm_device_types/host_shutdown.py b/napalm_device_types/host_shutdown.py new file mode 100644 index 0000000..e1e030b --- /dev/null +++ b/napalm_device_types/host_shutdown.py @@ -0,0 +1,32 @@ +"""Shutting the device itself down cleanly. + +Declared under ``if TYPE_CHECKING``: a contract, not a placeholder. Only a +driver that can shut its device down through the device's own means defines +``shutdown_host``, so ``hasattr(driver, "shutdown_host")`` tells a caller +whether to offer it. There is no forced variant here, by design: cutting the +power is the BMC's business, not a driver's. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + + +class HostShutdownMixin: + if TYPE_CHECKING: + + def shutdown_host(self) -> None: + """ + Shuts down the device this driver is connected to, cleanly. + + Returns once the device has accepted the request; the session is + usually gone right after. Whether the device really went off is + the caller's business, e.g. through its BMC. + + A driver that manages other machines shuts down its *own* host, + never one of them: a hypervisor node shuts down itself, and its + guests are stopped the way the node stops them on shutdown. + + :raises RuntimeError: If the device refuses the request. + """ + ... diff --git a/napalm_device_types/vm_stop_request.py b/napalm_device_types/vm_stop_request.py new file mode 100644 index 0000000..5d0b5de --- /dev/null +++ b/napalm_device_types/vm_stop_request.py @@ -0,0 +1,29 @@ +"""Asking for a VM's clean shutdown without waiting for it. + +``HypervisorDriver.stop_vm`` blocks until the VM is stopped, within a limit +of the driver's own. A caller that keeps its waits as timers asks with +``begin_stop_vm`` and reads ``get_vm_status`` until the VM says it stopped. + +Declared under ``if TYPE_CHECKING``: a driver mixes this in when its +hypervisor can take the request on its own; ``hasattr`` tells a caller +whether it did. No role base carries it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + + +class VmStopRequestMixin: + if TYPE_CHECKING: + + def begin_stop_vm(self, name: str) -> None: + """ + Asks the hypervisor to shut a virtual machine down cleanly (ACPI) + and returns as soon as the request was accepted. + + :param name: VM name or its ``vmid``. + :raises ValueError: If no VM with the given name/ID exists. + :raises RuntimeError: If the hypervisor refuses the request. + """ + ... diff --git a/tests/test_host_shutdown.py b/tests/test_host_shutdown.py new file mode 100644 index 0000000..0a9182a --- /dev/null +++ b/tests/test_host_shutdown.py @@ -0,0 +1,27 @@ +"""shutdown_host: the contract for shutting the device itself down cleanly. + +netOrk's graph workflows switch a hypervisor off when it has nothing to do +(NetOrk/netork#1636): through its own API, never by pulling the power. Like +``reboot_host`` it is declared, not implemented, so ``hasattr`` tells netOrk +whether a driver can. +""" + +from __future__ import annotations + +import inspect + +from napalm_device_types import DeviceTypeDriver, HostShutdownMixin + + +def test_every_device_type_driver_carries_the_declaration(): + assert issubclass(DeviceTypeDriver, HostShutdownMixin) + + +def test_declared_not_implemented(): + assert not hasattr(DeviceTypeDriver, "shutdown_host") + + +def test_declaration_documents_the_contract(): + source = inspect.getsource(HostShutdownMixin) + assert "def shutdown_host(self) -> None" in source + assert "RuntimeError" in source diff --git a/tests/test_vm_stop_request.py b/tests/test_vm_stop_request.py new file mode 100644 index 0000000..144a3fc --- /dev/null +++ b/tests/test_vm_stop_request.py @@ -0,0 +1,28 @@ +"""begin_stop_vm: ask for a VM's clean shutdown and return at once. + +``stop_vm`` blocks until the VM is stopped, with a limit of the driver's own. +A caller that keeps its waits as timers (netOrk's graph workflows, +NetOrk/netork#1636) asks with ``begin_stop_vm`` and reads ``get_vm_status`` +until the VM is stopped. A driver mixes it in when it can; no role demands it. +""" + +from __future__ import annotations + +import inspect + +from napalm_device_types import HypervisorDriver, VmStopRequestMixin + + +def test_no_role_demands_it(): + assert not issubclass(HypervisorDriver, VmStopRequestMixin) + + +def test_declared_not_implemented(): + assert not hasattr(VmStopRequestMixin, "begin_stop_vm") + + +def test_declaration_documents_the_contract(): + source = inspect.getsource(VmStopRequestMixin) + assert "def begin_stop_vm(self, name: str) -> None" in source + assert "ValueError" in source + assert "RuntimeError" in source From fbd3cf0129a4ce0bcc8795c374889e5797c606ef Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 27 Oct 2026 10:18:55 +0100 Subject: [PATCH 23/50] chore: Python 3.12 is the floor (NAPALM/driver-standard#6) netOrk runs every driver in its own containers on Python 3.12, so 3.10 and 3.11 are no longer supported or tested. requires-python, mypy and ruff move to 3.12, along with what ruff's pyupgrade rules then rewrite. --- CHANGELOG.md | 4 ++++ README.md | 2 +- napalm_device_types/_matching.py | 9 ++++----- napalm_device_types/models.py | 4 ++-- pyproject.toml | 8 +++----- tests/test_fact_getters.py | 3 +-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94e7a17..8571d00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ recorded here; their history is the git log. ## [Unreleased] +### Changed +- Python 3.12 is the floor (`requires-python = ">=3.12"`): netOrk runs every driver + on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). + ### Added - `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself down cleanly through its own means; declared, so `hasattr` says whether a driver diff --git a/README.md b/README.md index 44c28e0..a206e3f 100644 --- a/README.md +++ b/README.md @@ -390,7 +390,7 @@ detection vs. presentation there. pip install napalm-device-types ``` -Requires Python ≥ 3.10 (tested up to 3.13) and NAPALM ≥ 4.0. +Requires Python ≥ 3.12 (tested up to 3.13) and NAPALM ≥ 4.0. ## Available base classes diff --git a/napalm_device_types/_matching.py b/napalm_device_types/_matching.py index 839c16e..5cab64d 100644 --- a/napalm_device_types/_matching.py +++ b/napalm_device_types/_matching.py @@ -8,12 +8,9 @@ name a single entry, and guessing would overwrite one the caller never meant. from __future__ import annotations from collections.abc import Callable, Iterable -from typing import TypeVar - -T = TypeVar("T") -def index_live(entries: Iterable[T], key: Callable[[T], str]) -> tuple[dict[str, T], set[str]]: +def index_live[T](entries: Iterable[T], key: Callable[[T], str]) -> tuple[dict[str, T], set[str]]: """Live entries by key, and the keys more than one of them holds. Entries without a key are left out: they are not the caller's to match. @@ -32,7 +29,9 @@ def index_live(entries: Iterable[T], key: Callable[[T], str]) -> tuple[dict[str, return by_key, shared -def check_desired(entries: list[T], key: Callable[[T], str], shared: set[str], what: str) -> None: +def check_desired[T]( + entries: list[T], key: Callable[[T], str], shared: set[str], what: str +) -> None: """``ValueError`` for a desired entry with no key, a key given twice, or a key that names more than one live entry.""" seen: set[str] = set() diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 6b4effc..9dbb158 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -5,9 +5,9 @@ These supplement the types defined in napalm.base.models and are used by the abstract device-type driver classes in this package. """ -from typing import Literal, get_args +from typing import Literal, NotRequired, get_args -from typing_extensions import NotRequired, TypedDict +from typing_extensions import TypedDict # --------------------------------------------------------------------------- # Common (shared across device types) diff --git a/pyproject.toml b/pyproject.toml index 66b9c88..7f865d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "napalm-device-types" version = "4.22.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.12" license = "Apache-2.0" authors = [ { name = "Christian Manivong", email = "christian@manivong.de" }, @@ -30,8 +30,6 @@ classifiers = [ "Intended Audience :: System Administrators", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: System :: Networking", @@ -65,7 +63,7 @@ napalm_device_types = ["py.typed"] [tool.ruff] line-length = 100 -target-version = "py310" +target-version = "py312" [tool.ruff.lint] select = ["E", "F", "I", "UP", "B"] @@ -75,7 +73,7 @@ select = ["E", "F", "I", "UP", "B"] "tests/**" = ["E501"] [tool.mypy] -python_version = "3.10" +python_version = "3.12" strict = true ignore_missing_imports = true diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py index ac820fa..d9cc556 100644 --- a/tests/test_fact_getters.py +++ b/tests/test_fact_getters.py @@ -30,8 +30,7 @@ def test_a_missing_license_may_say_what_the_device_withholds_without_it(): # A Supermicro X10 with only SFT-OOB-LIC answers Redfish but keeps its drives and # firmware inventory until DCMS is activated: partly licensed, not unlicensed. hints = get_type_hints(LicenseDict) - # Python 3.10 keeps NotRequired[...] around the type; later versions strip it. - assert "list[str]" in str(hints["withholds"]) + assert hints["withholds"] == list[str] assert "withholds" not in LicenseDict.__required_keys__ From 1cef3385e8368d78290e1e85d6dd7b327fe03edd Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 27 Oct 2026 10:49:27 +0100 Subject: [PATCH 24/50] fix: the floor on napalm is 5.1 (NAPALM/driver-standard#6) napalm 5.0 and older import pkg_resources, which a Python 3.12 venv no longer has, so the lowest job failed on them. --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8571d00..a52306f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ recorded here; their history is the git log. ### Changed - Python 3.12 is the floor (`requires-python = ">=3.12"`): netOrk runs every driver on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). +- The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python + 3.12 venv no longer has. ### Added - `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself diff --git a/pyproject.toml b/pyproject.toml index 7f865d2..06800cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "napalm>=4.0", + "napalm>=5.1", # older releases import pkg_resources, which a 3.12 venv lacks "typing-extensions>=4.0", ] From 874c6c5a92aff65ca7c04bf9ef20b4c00f50d22c Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 30 Oct 2026 08:37:08 +0100 Subject: [PATCH 25/50] feat: ARP_AGE_UNKNOWN, the age of a neighbour the device does not report NAPALM requires an age on every entry of get_arp_table and get_ipv6_neighbors_table. Many devices do not report one, and drivers filled in 0.0, which means "confirmed just now". ARP_AGE_UNKNOWN (-1.0, as NAPALM's IOS driver reports) is the value for "not reported"; the README states the contract. NetOrk/netork#1327 --- CHANGELOG.md | 4 ++++ README.md | 11 +++++++++++ napalm_device_types/__init__.py | 2 ++ napalm_device_types/arp.py | 13 +++++++++++++ tests/test_arp_age.py | 27 +++++++++++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 napalm_device_types/arp.py create mode 100644 tests/test_arp_age.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a52306f..7fcabad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports + in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how + old an entry is. `0.0` means "confirmed just now" and is no stand-in for it + (NetOrk/netork#1327). - `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself down cleanly through its own means; declared, so `hasattr` says whether a driver can. No forced variant (NetOrk/netork#1636). diff --git a/README.md b/README.md index a206e3f..c982f43 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,17 @@ base; a mixin listed after it loses the MRO to NAPALM's stub. For the same reaso `hasattr` says nothing about NAPALM's getters, and the OS role's account listing is `get_os_users`, not `get_users` -- NAPALM's `get_users` has a different shape. +### NAPALM's getters: an unknown ARP age is `-1.0` + +NAPALM requires an `age` on every entry of `get_arp_table` and +`get_ipv6_neighbors_table`: seconds since the device last confirmed that neighbour. +Many devices do not report one (`ip neigh`, `/proc/net/arp`, a router's host list). +A driver that cannot read the age reports `ARP_AGE_UNKNOWN` (`-1.0`, as NAPALM's IOS +driver does), never `0.0`: `0.0` means "confirmed just now", and an application that +ranks sightings by freshness takes it at its word (NetOrk/netork#1327). An expiry +countdown (FreeBSD's `expires in 1180 seconds`) is not an age either. A driver whose +device does report the age keeps reporting it. + ## Function classes: what a device *can do* Behaviour shared across roles lives in a function class, exactly once, and a role base diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 9cc53bf..8945a20 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -74,6 +74,7 @@ from napalm_device_types.apt_proxy import ( parse_apt_proxy, same_proxy_url, ) +from napalm_device_types.arp import ARP_AGE_UNKNOWN from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec, rules_max_weight from napalm_device_types.channel import ( ByteStream, @@ -278,6 +279,7 @@ __all__ = [ "InterfaceFilterMixin", "MacAclMixin", "ManagementControllerDriver", + "ARP_AGE_UNKNOWN", "POWER_ACTIONS", "PowerAction", "PowerStateDict", diff --git a/napalm_device_types/arp.py b/napalm_device_types/arp.py new file mode 100644 index 0000000..c543a89 --- /dev/null +++ b/napalm_device_types/arp.py @@ -0,0 +1,13 @@ +"""The ``age`` of an entry in NAPALM's neighbour tables when the device does not say. + +NAPALM requires an ``age`` on every entry of ``get_arp_table`` and +``get_ipv6_neighbors_table``: seconds since the device last confirmed that +neighbour. Many devices do not report one. ``0.0`` means "confirmed just now" +and must never stand in for "not reported": a caller ranking sightings by +freshness takes it at its word (NetOrk/netork#1327). An expiry countdown is not +an age either. The value is NAPALM's own; its IOS driver reports ``-1.0``. +""" + +from typing import Final + +ARP_AGE_UNKNOWN: Final[float] = -1.0 diff --git a/tests/test_arp_age.py b/tests/test_arp_age.py new file mode 100644 index 0000000..d913dd2 --- /dev/null +++ b/tests/test_arp_age.py @@ -0,0 +1,27 @@ +"""An ARP or neighbour entry whose age the device does not report says so. + +NAPALM's ``get_arp_table`` and ``get_ipv6_neighbors_table`` require an ``age`` +on every entry, and a driver that could not read one used to fill in ``0.0``. +To a caller that is the freshest sighting there is: netOrk took a neighbour the +device had not confirmed for hours as seen this second (NetOrk/netork#1327). +Unknown is ``-1.0``, as in NAPALM's own IOS driver; ``0.0`` means "just now". +""" + +from __future__ import annotations + +from napalm_device_types import ARP_AGE_UNKNOWN +from napalm_device_types.arp import ARP_AGE_UNKNOWN as FROM_MODULE + + +def test_an_unknown_age_is_minus_one(): + assert ARP_AGE_UNKNOWN == -1.0 + assert isinstance(ARP_AGE_UNKNOWN, float) + + +def test_the_package_exports_the_modules_constant(): + assert ARP_AGE_UNKNOWN is FROM_MODULE + + +def test_no_real_age_can_be_mistaken_for_unknown(): + # An age is seconds since the entry was last confirmed: never negative. + assert ARP_AGE_UNKNOWN < 0 From a5b7b540c6d6c10ba5709a1d8392cedc6686b858 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 30 Oct 2026 09:27:23 +0100 Subject: [PATCH 26/50] feat: NetworkTargetDict.uplinks, the NICs a bridge leaves through (NetOrk/netork#1325) A VM on a VLAN-aware bridge without a VLAN tag only gets an address when the switch port behind the bridge carries an untagged network. To find that port, the caller needs the bridge's physical NICs: their LLDP neighbours name the switch and the port. uplinks lists them in groups, one per way out of the bridge: a bond's slaves together, a single NIC alone. A bond is one link, so a caller can let one slave's neighbour speak for the others when lldpd sees only one. Internal, VLAN and virtual ports are no uplinks. A missing key means the driver cannot tell; an empty list is a bridge without uplink. --- CHANGELOG.md | 6 ++++++ README.md | 8 ++++++++ napalm_device_types/hypervisor.py | 4 +++- napalm_device_types/models.py | 10 +++++----- tests/test_network_target_uplinks.py | 25 +++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 tests/test_network_target_uplinks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a52306f..50d1cc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `NetworkTargetDict.uplinks` (optional): the physical NICs a bridge leaves the + host through, one group per way out -- a bond's slaves together, a single NIC + alone (`[["eno1", "eno2"]]`). A missing key means unknown, an empty list a + bridge without uplink. NetOrk follows their LLDP neighbours to the switch port + and its untagged VLAN; one slave's neighbour speaks for its bond + (NetOrk/netork#1325). - `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself down cleanly through its own means; declared, so `hasattr` says whether a driver can. No forced variant (NetOrk/netork#1636). diff --git a/README.md b/README.md index a206e3f..cf369a6 100644 --- a/README.md +++ b/README.md @@ -503,6 +503,14 @@ class ProxmoxDriver(HypervisorDriver): def create_vm_snapshot(self, name, snapshot, description="", include_memory=False): ... + def get_network_targets(self): + # return List[NetworkTargetDict]: bridges and vnets a new VM's NIC may + # attach to; a bridge may name its physical NICs in `uplinks`, one group + # per way out: [["eno1", "eno2"]] for a bond, [["eno1"]] for a single NIC, + # no internal or VLAN ports -- leave the key out when they cannot be + # determined + ... + def get_vm_cpu_types(self): # optional — return List[VMCpuTypeDict]: the CPU models a new VM may get # on this node, each with its cpuinfo flags and whether the node can run diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 5406f1d..86c3d42 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -613,7 +613,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri List[NetworkTargetDict] - each entry's ``vlan_aware`` flag tells the caller whether a ``NICConfigDict.vlan_tag`` may additionally be set for a NIC using that target (see ``NetworkTargetDict`` for the - per-kind rules). + per-kind rules). ``uplinks``, where the driver can tell, names + the physical NICs a bridge leaves the host through, grouped: + a bond's slaves in one group, a single NIC in its own. """ ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 9dbb158..112a57c 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1310,17 +1310,17 @@ class NetworkTargetDict(TypedDict): Distinguishes real bridges (Linux or OVS) from SDN network segments (vnets), and tells the caller whether a separate ``vlan_tag`` may be applied on top: - Linux bridge: vlan_aware reflects the bridge's own ``bridge_vlan_aware`` flag. - - OVS bridge: always vlan_aware (OVS bridges tag per-port regardless of a - dedicated "VLAN aware" setting). - - SDN vnet: never vlan_aware — the VLAN is already fixed by the vnet's zone/tag, - so a NIC attached to it must not also carry a ``vlan_tag``. That fixed VLAN - is surfaced via ``fixed_vlan_tag`` instead, for display purposes. + - OVS bridge: always vlan_aware (OVS tags per port, with no "VLAN aware" setting). + - SDN vnet: never vlan_aware — its zone/tag fixes the VLAN (``fixed_vlan_tag``). + ``uplinks``: the physical NICs it leaves the host through, one group per way out + (a bond's slaves together, a single NIC alone); no internal, VLAN or virtual ports. """ name: str # Bridge, vnet or port group name, usable directly as NICConfigDict.bridge kind: str # "bridge", "vnet" or "portgroup" (VMware: VLAN fixed like a vnet's) vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of this target fixed_vlan_tag: NotRequired[int | None] # vnet only: the VLAN ID already baked into it + uplinks: NotRequired[list[list[str]]] # NIC groups it leaves through; absent = unknown class StorageTargetDict(TypedDict): diff --git a/tests/test_network_target_uplinks.py b/tests/test_network_target_uplinks.py new file mode 100644 index 0000000..bfc045b --- /dev/null +++ b/tests/test_network_target_uplinks.py @@ -0,0 +1,25 @@ +"""A network target can say which physical NICs it leaves the host through. + +A VM on a VLAN-aware bridge without a VLAN tag only gets an address when the +switch port behind the bridge carries an untagged network. To look that port up, +the caller needs the bridge's physical NICs: their LLDP neighbours name the +switch and the port (NetOrk/netork#1325). They come in groups, one per way out +of the bridge: a bond's slaves together, a single NIC alone. A bond has no +neighbour of its own, and its slaves are one link, so one slave's neighbour +speaks for the others when they report none. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types.models import NetworkTargetDict + + +def test_uplinks_is_an_optional_list_of_nic_groups(): + assert "uplinks" in NetworkTargetDict.__optional_keys__ + assert get_type_hints(NetworkTargetDict)["uplinks"] == list[list[str]] + + +def test_contract_core_stays_required(): + assert {"name", "kind", "vlan_aware"} <= NetworkTargetDict.__required_keys__ From cb90fa9835296a53b35c61956a128542666274e3 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 30 Oct 2026 21:09:24 +0100 Subject: [PATCH 27/50] feat: SudoHistoryMixin, the sudo password left in shell histories (NetOrk/netork#1090) A driver that typed `echo '' | sudo -S -p "" ` into an interactive shell left that line, password and all, in the login user's shell history once per command, and where only the part before the first `&&` ran as root, files root should own were left to the login user. get_history_sudo_lines() counts the lines per history file with grep -c, as the login user, without reading any back. remove_history_sudo_lines() deletes exactly those lines with sed -i, overwrites the backup with shred -u, and as root gives the driver's SUDO_ROOT_FILES back to root:root where each is a regular file, not a link, owned by the login user. It reports each file's lines before and after. Nothing goes to stdin. The driver names the marker and the files and supplies run_command. --- CHANGELOG.md | 13 + README.md | 10 + napalm_device_types/__init__.py | 11 + napalm_device_types/sudo_history.py | 312 ++++++++++++++++++++++ tests/test_sudo_history.py | 394 ++++++++++++++++++++++++++++ 5 files changed, 740 insertions(+) create mode 100644 napalm_device_types/sudo_history.py create mode 100644 tests/test_sudo_history.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ff6b8b1..1b04273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,19 @@ recorded here; their history is the git log. - `VmStopRequestMixin.begin_stop_vm(name)`: ask for a VM's clean shutdown and return at once, for callers that wait on a timer instead of in `stop_vm`. No role demands it (NetOrk/netork#1636). +- `SudoHistoryMixin` (`napalm_device_types.sudo_history`): the sudo password an earlier + driver typed into an interactive shell (`echo '' | sudo -S -p "" …`) and + left in the login user's shell history, for a driver to mix in (NetOrk/netork#1090). + The driver says what its lines carry (`SUDO_HISTORY_MARKER`) and which files its + partial sudo left to the login user (`SUDO_ROOT_FILES`), and supplies `run_command`. + - `get_history_sudo_lines()` counts the lines per history file (`~/.bash_history`, + `~/.zsh_history`, `~/.ash_history`, `~/.sh_history`) with `grep -c` as the login + user. No line is read back; a file it cannot read raises. + - `remove_history_sudo_lines()` deletes exactly those lines as the login user (`sed + -i`, its backup overwritten with `shred -u`), and as root gives each of + `SUDO_ROOT_FILES` back to `root:root` where it is a regular file, not a symbolic + link, owned by the login user. It reports every file's lines before and after, + how many were removed and which files are root's again. Nothing goes to stdin. ## [4.22.0] - 2026-10-23 diff --git a/README.md b/README.md index 4705be7..8a4c22e 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,16 @@ A driver mixes it in and supplies `run_command`; a platform that regenerates ssh itself (an appliance with its own database) must not, or opts out with `harden_ssh_server = None`. +`SudoHistoryMixin` (`get_history_sudo_lines`, `remove_history_sudo_lines`) deals with what +a driver left behind when it typed `echo '' | sudo -S -p "" …` into an interactive +shell: that line, password and all, in the login user's shell history, and files its partial +sudo wrote as the login user. The driver names the marker its lines carry +(`SUDO_HISTORY_MARKER`) and those files (`SUDO_ROOT_FILES`). The getter counts the lines per +history file with `grep -c`, as the login user; the fix deletes them with `sed -i` and +overwrites the backup with `shred -u`, then gives the files back to root (regular files the +login user owns, never a symbolic link). The password is never read, sent or written. A +driver mixes it in and supplies `run_command`. + **Update readers raise when they cannot read.** `get_available_updates` returns an empty list only when nothing is pending; netOrk keeps "pending since" per package, and an empty list for "don't know" would reset it. diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 8945a20..b61a8ae 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -55,6 +55,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.ping_sweep.PingSweepMixin` * :class:`~napalm_device_types.service_clusters.ServiceClustersMixin` * :class:`~napalm_device_types.service_packages.ServicePackagesMixin` +* :class:`~napalm_device_types.sudo_history.SudoHistoryMixin` * :class:`~napalm_device_types.services.ServiceControlMixin` * :class:`~napalm_device_types.systemd.SystemdServicesMixin` * :class:`~napalm_device_types.updates.UpdateMixin` @@ -198,6 +199,12 @@ from napalm_device_types.ssh_server import ( plan_ssh_hardening, ) from napalm_device_types.storage import StorageDriver +from napalm_device_types.sudo_history import ( + HistoryCleanupDict, + HistorySudoLinesDict, + SudoHistoryCleanupDict, + SudoHistoryMixin, +) from napalm_device_types.switch import SwitchDriver from napalm_device_types.system_identity import ( SYSTEM_IDENTITY_COMMAND, @@ -338,6 +345,10 @@ __all__ = [ "SshHardeningPlan", "SshHardeningResult", "plan_ssh_hardening", + "SudoHistoryMixin", + "HistorySudoLinesDict", + "HistoryCleanupDict", + "SudoHistoryCleanupDict", "SSH_HOST_KEYS_ARG", "SSH_HOST_KEY_OBSERVER_ARG", "HostKeyCheck", diff --git a/napalm_device_types/sudo_history.py b/napalm_device_types/sudo_history.py new file mode 100644 index 0000000..865adb6 --- /dev/null +++ b/napalm_device_types/sudo_history.py @@ -0,0 +1,312 @@ +"""The sudo password an earlier driver left in shell histories, and taking it back. + +A driver that typed ``echo '' | sudo -S -p "" `` into an +interactive shell left that line, password and all, in the login user's shell +history, once per command -- and where only the part before the first ``&&`` +ran as root, it left files root should own to the login user. A driver that has +stopped doing so still leaves both behind on every host it ever reached. + +:class:`SudoHistoryMixin` reads the first as a fact and takes both back. The +commands are the same on every POSIX host, so they live here once; a driver +mixes it in, supplies ``run_command`` and says what it once left behind: +:attr:`~SudoHistoryMixin.SUDO_HISTORY_MARKER` and +:attr:`~SudoHistoryMixin.SUDO_ROOT_FILES`. + +**The password never travels.** Lines are counted with ``grep -c`` and deleted +with ``sed``, both matching the marker only, as the login user who owns the +history. Nothing of a line's content is read back, and nothing goes to stdin. + +**What is deleted is gone from the disk too, as far as a file can say.** ``sed +-i`` with a backup suffix renames the old file to the backup and writes a new +one; the backup -- the old file's blocks -- is overwritten with ``shred -u``, +or removed with ``rm`` where there is no ``shred``. Backups, snapshots and an +audit log of the host may still hold the password: only rotating it ends that. + +**A file given back to root** must exist as a regular file, not be a symbolic +link, and be owned by the login user. ``chown -h`` never follows a link that +appears in between. +""" + +from __future__ import annotations + +from collections.abc import Callable +from shlex import quote +from typing import TYPE_CHECKING, ClassVar + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepDict, FixStepOutcomeDict + +if TYPE_CHECKING: # pragma: no cover - for type checkers only + from napalm_device_types.channel import CommandResult + +#: The shell history files looked at, in the login user's home. +HISTORY_FILES = (".bash_history", ".zsh_history", ".ash_history", ".sh_history") + +#: The suffix of the moment's backup ``sed -i`` makes; it carries no application's name. +HISTORY_BACKUP_SUFFIX = ".napalm-bak" + +_COUNT = 'LC_ALL=C grep -c -F -e "$m" -- "$p"' + + +class HistorySudoLinesDict(TypedDict): + """A history file holding lines with the sudo password (``get_history_sudo_lines``).""" + + #: The file, as ``~/``. + file: str + lines: int + + +class HistoryCleanupDict(TypedDict): + """What one history file held before the cleanup and holds after it.""" + + file: str + before: int + after: int + + +class SudoHistoryCleanupDict(FixResultDict): + """The result of ``remove_history_sudo_lines``: the fix, and what it changed.""" + + #: Every history file there is, with its lines before and after. + files: list[HistoryCleanupDict] + #: How many lines were deleted, over all files. + removed: int + #: The files given back to root. + owners_restored: list[str] + + +def _bre(text: str) -> str: + """*text* as a POSIX basic regular expression matching it literally, for ``sed``.""" + return "".join("\\" + c if c in "\\.[]*^$/" else c for c in text) + + +def _history_loop(body: str) -> str: + names = " ".join(HISTORY_FILES) + return f'for f in {names}; do p="$HOME/$f"; [ -f "$p" ] || continue; {body}; done' + + +def count_script(marker: str) -> str: + """Print `` `` per history file there is; exit 2 when one cannot be read.""" + return f"m={quote(marker)}; " + _history_loop( + f'n=$({_COUNT}); [ $? -le 1 ] || exit 2; printf "%s %s\\n" "$n" "$f"' + ) + + +def cleanup_script(marker: str) -> str: + """Delete the lines holding *marker* from every history file there is. + + Prints per file: ``done ``, ``link `` + (left alone), ``unreadable `` or ``failed ``. + """ + backup = f'"$p"{quote(HISTORY_BACKUP_SUFFIX)}' + return f"m={quote(marker)}; r={quote(_bre(marker))}; " + _history_loop( + f"b=$({_COUNT}); " + '[ $? -le 1 ] || { printf "unreadable %s\\n" "$f"; continue; }; ' + 'if [ -L "$p" ]; then printf "link %s %s\\n" "$b" "$f"; continue; fi; ' + 'if [ "$b" -gt 0 ]; then ' + f'LC_ALL=C sed -i{quote(HISTORY_BACKUP_SUFFIX)} -e "/$r/d" "$p"; s=$?; ' + # A sed that renamed the file away and wrote no new one: the backup is the history. + f'[ -e "$p" ] || mv -f {backup} "$p"; ' + f"[ -e {backup} ] && {{ shred -u {backup} 2>/dev/null || rm -f {backup}; }}; " + '[ $s -eq 0 ] || { printf "failed %s %s\\n" "$b" "$f"; continue; }; ' + "fi; " + f'a=$({_COUNT}); printf "done %s %s %s\\n" "$b" "$a" "$f"' + ) + + +def owners_script(user: str, paths: tuple[str, ...]) -> str: + """As root: give each of *paths* back to root:root where *user* owns it. + + Prints ``fixed `` or ``failed `` per file it changed or tried to. + """ + return ( + f"u={quote(user)}; for p in {' '.join(quote(p) for p in paths)}; do " + '[ -f "$p" ] && [ ! -L "$p" ] || continue; ' + '[ -n "$(find "$p" -prune -user "$u" 2>/dev/null)" ] || continue; ' + 'if chown -h 0:0 "$p"; then printf "fixed %s\\n" "$p"; ' + 'else printf "failed %s\\n" "$p"; fi; done' + ) + + +def _home(name: str) -> str: + return f"~/{name}" + + +def _lines(n: int) -> str: + return "1 line" if n == 1 else f"{n} lines" + + +def _done(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + before, after, name = int(rest[0]), int(rest[1]), _home(rest[2]) + removed = before - after + text = ( + f"{name}: {_lines(removed)} removed ({before} before, {after} after)" + if before + else f"{name}: no such line" + ) + return after == 0, text, {"file": name, "before": before, "after": after} + + +def _link(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + count = int(rest[0]) + return count == 0, f"{_home(rest[1])}: a symbolic link, left alone ({_lines(count)})", None + + +def _unreadable(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + return False, f"{_home(rest[0])}: cannot be read", None + + +def _failed(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + kept = _lines(int(rest[0])) + return False, f"{_home(rest[1])}: could not be rewritten ({kept} kept)", None + + +#: Per kind of line :func:`cleanup_script` prints: how many words follow, and what it says. +_CLEANUP_LINES: dict[ + str, tuple[int, Callable[[list[str]], tuple[bool, str, HistoryCleanupDict | None]]] +] = { + "done": (3, _done), + "link": (2, _link), + "unreadable": (1, _unreadable), + "failed": (2, _failed), +} + + +class SudoHistoryMixin: + """Reads and removes the sudo password an earlier driver left in shell histories. + + Mixed in by the drivers that once typed it into a shell, not by a base + class, so ``callable(getattr(driver, "get_history_sudo_lines", None))`` + stays a truthful answer. + """ + + #: What every line carries that typed the sudo password into the shell, as a + #: fixed string -- ``"| sudo -S -p "`` for ``echo '' | sudo -S -p "" + #: ``. None for a driver that never did: nothing is looked for. + SUDO_HISTORY_MARKER: ClassVar[str | None] = None + + #: Files an earlier version wrote as the login user where root should have. + SUDO_ROOT_FILES: ClassVar[tuple[str, ...]] = () + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: ... + + def get_history_sudo_lines(self) -> list[HistorySudoLinesDict]: + """The login user's history files that hold lines with the sudo password, and how many. + + A fact: counted with ``grep -c`` as the login user; no line is read. + An empty list when no file holds one, or the driver never typed it. + + :raises RuntimeError: when a history file cannot be read. + """ + marker = self.SUDO_HISTORY_MARKER + if not marker: + return [] + result = self.run_command(count_script(marker), timeout=30) + if result.exit_code != 0: + raise RuntimeError( + f"cannot count the shell history: {(result.stderr or result.stdout).strip()}" + ) + found: list[HistorySudoLinesDict] = [] + for line in result.stdout.splitlines(): + count, _, name = line.strip().partition(" ") + if count.isdigit() and int(count) and name: + found.append({"file": _home(name), "lines": int(count)}) + return found + + def remove_history_sudo_lines(self) -> SudoHistoryCleanupDict: + """Delete those lines, as the login user, and give :attr:`SUDO_ROOT_FILES` back to root. + + Two steps, ``history`` and ``owners``; the second runs when the first + failed, too. Succeeds when no history file holds such a line any more + and every file the login user owned is root's again. + """ + files: list[HistoryCleanupDict] = [] + restored: list[str] = [] + parts = [ + run_fix_steps([("history", lambda: self._clean_histories(files))]), + run_fix_steps([("owners", lambda: self._restore_owners(restored))]), + ] + steps: list[FixStepDict] = [s for part in parts for s in part["steps"]] + return { + "success": all(part["success"] for part in parts), + "steps": steps, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in steps), + "files": files, + "removed": sum(f["before"] - f["after"] for f in files), + "owners_restored": restored, + } + + def _clean_histories(self, files: list[HistoryCleanupDict]) -> FixStepOutcomeDict: + marker = self.SUDO_HISTORY_MARKER + if not marker: + return {"ok": True, "output": "this driver never typed the sudo password"} + result = self.run_command(cleanup_script(marker), timeout=60) + said: list[str] = [] + ok = result.exit_code == 0 + for line in result.stdout.splitlines(): + ok = self._history_line(line.split(), files, said) and ok + if result.exit_code != 0: + said.append((result.stderr or result.stdout).strip()) + return {"ok": ok, "output": "\n".join(said) or "no shell history"} + + @staticmethod + def _history_line(words: list[str], files: list[HistoryCleanupDict], said: list[str]) -> bool: + """Take one line of :func:`cleanup_script` into *files* and *said*; whether it is clean.""" + kind, rest = (words[0], words[1:]) if words else ("", []) + read = _CLEANUP_LINES.get(kind) + if read is None or len(rest) != read[0]: + return True + clean, text, entry = read[1](rest) + said.append(text) + if entry is not None: + files.append(entry) + return clean + + def _restore_owners(self, restored: list[str]) -> FixStepOutcomeDict: + paths = tuple(self.SUDO_ROOT_FILES) + if not paths: + return {"ok": True, "output": "no file to check"} + who = self.run_command("id -u; id -un", timeout=30).stdout.split() + if len(who) < 2: + return {"ok": False, "output": "cannot tell who the login user is"} + uid, user = who[0], who[1] + if uid == "0": + return {"ok": True, "output": "logged in as root: every file it wrote is root's"} + result = self.run_command(owners_script(user, paths), privileged=True, timeout=30) + said: list[str] = [] + ok = result.exit_code == 0 + for line in result.stdout.splitlines(): + kind, _, path = line.strip().partition(" ") + if kind == "fixed": + restored.append(path) + said.append(f"{path}: owned by root again") + elif kind == "failed": + ok = False + said.append(f"{path}: could not be given back to root") + if result.exit_code != 0: + said.append((result.stderr or result.stdout).strip()) + return {"ok": ok, "output": "\n".join(said) or f"no file owned by {user}"} + + +__all__ = [ + "HISTORY_BACKUP_SUFFIX", + "HISTORY_FILES", + "HistoryCleanupDict", + "HistorySudoLinesDict", + "SudoHistoryCleanupDict", + "SudoHistoryMixin", + "cleanup_script", + "count_script", + "owners_script", +] diff --git a/tests/test_sudo_history.py b/tests/test_sudo_history.py new file mode 100644 index 0000000..d7415c0 --- /dev/null +++ b/tests/test_sudo_history.py @@ -0,0 +1,394 @@ +"""The sudo password an earlier driver left in shell histories, and taking it back. + +The scripts are run for real, with ``sh``, against a home directory in a temp +directory: what they count, delete and report is what a host would see. Root's +part -- giving files back to root -- runs with a ``chown`` that only records, +so the test needs no root. +""" + +from __future__ import annotations + +import getpass +import os +import subprocess +from pathlib import Path + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.sudo_history import HISTORY_BACKUP_SUFFIX, SudoHistoryMixin + +MARKER = "| sudo -S -p " + +LEAKED = "echo 'hunter2' | sudo -S -p \"\" apt-get update\n" +OTHER = "ls -la\n" + + +class _Host(SudoHistoryMixin): + """A driver whose host is this machine, with *home* as the login user's home.""" + + SUDO_HISTORY_MARKER = MARKER + + def __init__(self, home: Path, bin_dir: Path, *, root: bool = False) -> None: + self.home = home + self.bin_dir = bin_dir + self.root = root + #: Who ``id -un`` says the login user is; None for this machine's answer. + self.user: str | None = None + self.ran: list[tuple[str, bool]] = [] + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: + # The password goes ahead of a privileged command's stdin: there must be none. + assert not (privileged and stdin is not None) + self.ran.append((command, privileged)) + env = { + **os.environ, + "HOME": str(self.home), + "PATH": f"{self.bin_dir}:{os.environ['PATH']}", + } + if command.strip() == "id -u; id -un" and self.root: + return CommandResult("0\nroot\n", "", 0) + if command.strip() == "id -u; id -un" and self.user: + return CommandResult(f"4242\n{self.user}\n", "", 0) + done = subprocess.run( # noqa: S603 - the test's own scripts + ["sh", "-c", command], capture_output=True, text=True, env=env, timeout=timeout + ) + return CommandResult(done.stdout, done.stderr, done.returncode) + + +@pytest.fixture +def bin_dir(tmp_path: Path) -> Path: + """A ``chown`` that records its arguments instead of changing anything.""" + d = tmp_path / "bin" + d.mkdir() + chown = d / "chown" + chown.write_text(f'#!/bin/sh\nprintf "%s\\n" "$*" >> {tmp_path / "chowned"}\n') + chown.chmod(0o755) + return d + + +@pytest.fixture +def home(tmp_path: Path) -> Path: + h = tmp_path / "home" + h.mkdir() + return h + + +def _chowned(tmp_path: Path) -> list[str]: + log = tmp_path / "chowned" + return log.read_text().splitlines() if log.exists() else [] + + +class TestCounting: + def test_counts_the_lines_per_history_file_without_reading_them_out(self, home, bin_dir): + (home / ".bash_history").write_text(OTHER + LEAKED * 3) + (home / ".zsh_history").write_text(": 1700000000:0;" + LEAKED) + (home / ".sh_history").write_text(OTHER) + host = _Host(home, bin_dir) + + found = host.get_history_sudo_lines() + + assert found == [ + {"file": "~/.bash_history", "lines": 3}, + {"file": "~/.zsh_history", "lines": 1}, + ] + # Counted as the login user, and nothing of the content came back. + assert all(not privileged for _cmd, privileged in host.ran) + assert "hunter2" not in repr(found) + + def test_no_history_no_lines(self, home, bin_dir): + assert _Host(home, bin_dir).get_history_sudo_lines() == [] + + def test_the_pattern_is_a_fixed_string(self, home, bin_dir): + # "-S" is no regex trouble, but "|" would be an alternation in an ERE. + (home / ".bash_history").write_text("sudo -S -p\n| sudo\n") + assert _Host(home, bin_dir).get_history_sudo_lines() == [] + + def test_a_driver_that_never_typed_the_password_counts_nothing(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + host.SUDO_HISTORY_MARKER = None + + assert host.get_history_sudo_lines() == [] + assert host.ran == [] + + def test_an_unreadable_history_raises_rather_than_reporting_none(self, home, bin_dir): + if os.geteuid() == 0: + pytest.skip("root reads every file") + history = home / ".bash_history" + history.write_text(LEAKED) + history.chmod(0) + try: + with pytest.raises(RuntimeError): + _Host(home, bin_dir).get_history_sudo_lines() + finally: + history.chmod(0o600) + + +class TestRemoving: + def test_deletes_exactly_those_lines_and_says_how_many(self, home, bin_dir): + (home / ".bash_history").write_text(OTHER + LEAKED + "cd /etc\n" + LEAKED) + (home / ".ash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + + result = host.remove_history_sudo_lines() + + assert result["success"] is True + assert (home / ".bash_history").read_text() == OTHER + "cd /etc\n" + assert (home / ".ash_history").read_text() == "" + assert result["removed"] == 3 + assert result["files"] == [ + {"file": "~/.bash_history", "before": 2, "after": 0}, + {"file": "~/.ash_history", "before": 1, "after": 0}, + ] + assert "~/.bash_history: 2 lines removed (2 before, 0 after)" in result["output"] + + def test_no_backup_with_the_password_is_left_behind(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + + _Host(home, bin_dir).remove_history_sudo_lines() + + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + assert not (home / f".bash_history{HISTORY_BACKUP_SUFFIX}").exists() + + def test_the_history_runs_as_the_login_user(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + + host.remove_history_sudo_lines() + + history_scripts = [(c, p) for c, p in host.ran if ".bash_history" in c] + assert history_scripts and all(not p for _c, p in history_scripts) + + def test_a_clean_history_is_left_as_it_is(self, home, bin_dir): + history = home / ".bash_history" + history.write_text(OTHER) + before = history.stat().st_ino + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is True + assert result["removed"] == 0 + assert result["files"] == [{"file": "~/.bash_history", "before": 0, "after": 0}] + assert history.stat().st_ino == before # not rewritten + + def test_a_history_that_is_a_link_is_left_alone_and_the_fix_fails( + self, home, bin_dir, tmp_path + ): + target = tmp_path / "elsewhere" + target.write_text(LEAKED) + (home / ".bash_history").symlink_to(target) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert target.read_text() == LEAKED + assert (home / ".bash_history").is_symlink() + assert "~/.bash_history: a symbolic link, left alone (1 line)" in result["output"] + + def test_the_lines_of_a_driver_that_never_typed_the_password_are_not_looked_for( + self, home, bin_dir + ): + host = _Host(home, bin_dir) + host.SUDO_HISTORY_MARKER = None + + result = host.remove_history_sudo_lines() + + assert result["success"] is True + assert result["removed"] == 0 + + def test_an_unreadable_history_fails_the_step(self, home, bin_dir): + if os.geteuid() == 0: + pytest.skip("root reads every file") + history = home / ".bash_history" + history.write_text(LEAKED) + history.chmod(0) + try: + result = _Host(home, bin_dir).remove_history_sudo_lines() + finally: + history.chmod(0o600) + + assert result["success"] is False + assert "~/.bash_history: cannot be read" in result["output"] + + def test_a_sed_that_fails_keeps_the_lines_and_leaves_no_backup(self, home, bin_dir): + (bin_dir / "sed").write_text( + '#!/bin/sh\nfor a; do f="$a"; done\ncp "$f" "$f.napalm-bak"\nexit 4\n' + ) + (bin_dir / "sed").chmod(0o755) + (home / ".bash_history").write_text(LEAKED * 2) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert "~/.bash_history: could not be rewritten (2 lines kept)" in result["output"] + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + + def test_a_sed_that_lost_the_file_gets_it_back(self, home, bin_dir): + (bin_dir / "sed").write_text( + '#!/bin/sh\nfor a; do f="$a"; done\nmv "$f" "$f.napalm-bak"\nexit 4\n' + ) + (bin_dir / "sed").chmod(0o755) + (home / ".bash_history").write_text(OTHER + LEAKED) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert (home / ".bash_history").read_text() == OTHER + LEAKED + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + + def test_a_script_that_dies_says_why(self, home, bin_dir): + host = _Host(home, bin_dir) + host.run_command = lambda command, **kw: CommandResult("", "sh: killed", 137) # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "sh: killed" in result["output"] + + def test_a_line_it_does_not_know_is_ignored(self): + files: list = [] + said: list[str] = [] + + assert SudoHistoryMixin._history_line(["what", "ever"], files, said) is True + assert SudoHistoryMixin._history_line([], files, said) is True + assert files == [] and said == [] + + +class TestOwners: + def _host(self, home, bin_dir, files): + host = _Host(home, bin_dir) + host.SUDO_ROOT_FILES = tuple(str(f) for f in files) + return host + + def test_a_file_the_login_user_owns_goes_back_to_root(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert _chowned(tmp_path) == [f"-h 0:0 {conf}"] + assert result["owners_restored"] == [str(conf)] + assert result["success"] is True + + def test_only_as_root_and_without_stdin(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + + host.remove_history_sudo_lines() + + privileged = [c for c, p in host.ran if p] + assert len(privileged) == 1 and "chown" in privileged[0] + # The login user's name is quoted into root's script, never taken from stdin. + assert getpass.getuser() in privileged[0] + + def test_a_link_or_a_missing_file_is_not_touched(self, home, bin_dir, tmp_path): + real = tmp_path / "real" + real.write_text("x") + link = tmp_path / "00proxy" + link.symlink_to(real) + + result = self._host(home, bin_dir, [link, tmp_path / "missing"]).remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["owners_restored"] == [] + assert result["success"] is True + + def test_a_file_someone_else_owns_is_not_touched(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + host.user = "nobody-here-4242" + + result = host.remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["owners_restored"] == [] + + def test_a_root_login_has_nothing_to_give_back(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + host.root = True + + result = host.remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["success"] is True + + def test_the_owners_are_set_even_when_the_history_could_not_be_cleaned( + self, home, bin_dir, tmp_path + ): + target = tmp_path / "elsewhere" + target.write_text(LEAKED) + (home / ".bash_history").symlink_to(target) + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert result["success"] is False + assert [s["step"] for s in result["steps"]] == ["history", "owners"] + assert result["owners_restored"] == [str(conf)] + + def test_without_knowing_the_login_user_nothing_changes_owner(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + real = host.run_command + + def run(command, **kw): + if command.startswith("id "): + return CommandResult("", "id: not found", 127) + return real(command, **kw) + + host.run_command = run # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "cannot tell who the login user is" in result["output"] + assert _chowned(tmp_path) == [] + + def test_a_sudo_that_refuses_fails_the_step(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + real = host.run_command + + def run(command, *, privileged=False, **kw): + if privileged: + return CommandResult("", "sudo: a password is required", 1) + return real(command, privileged=privileged, **kw) + + host.run_command = run # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "sudo: a password is required" in result["output"] + + def test_a_refused_chown_fails_the_step(self, home, bin_dir, tmp_path): + (bin_dir / "chown").write_text("#!/bin/sh\necho 'Operation not permitted' >&2\nexit 1\n") + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert result["success"] is False + assert result["owners_restored"] == [] + assert f"{conf}: could not be given back to root" in result["output"] + + +def test_exported_from_the_package(): + import napalm_device_types + + assert napalm_device_types.SudoHistoryMixin is SudoHistoryMixin From 0f0064c2d7d482f3cba2620d673862b142a4e7f5 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 30 Oct 2026 21:44:40 +0100 Subject: [PATCH 28/50] test: the fake host never answers with this machine's uid CI runs as root, which the owner step skips, so three owner tests failed there while they passed for a normal user. --- tests/test_sudo_history.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_sudo_history.py b/tests/test_sudo_history.py index d7415c0..76d62e6 100644 --- a/tests/test_sudo_history.py +++ b/tests/test_sudo_history.py @@ -33,7 +33,7 @@ class _Host(SudoHistoryMixin): self.home = home self.bin_dir = bin_dir self.root = root - #: Who ``id -un`` says the login user is; None for this machine's answer. + #: Who ``id -un`` says the login user is; None for this machine's user name. self.user: str | None = None self.ran: list[tuple[str, bool]] = [] @@ -55,8 +55,9 @@ class _Host(SudoHistoryMixin): } if command.strip() == "id -u; id -un" and self.root: return CommandResult("0\nroot\n", "", 0) - if command.strip() == "id -u; id -un" and self.user: - return CommandResult(f"4242\n{self.user}\n", "", 0) + if command.strip() == "id -u; id -un": + # Never this machine's uid: CI runs as root, which the owner step skips. + return CommandResult(f"4242\n{self.user or getpass.getuser()}\n", "", 0) done = subprocess.run( # noqa: S603 - the test's own scripts ["sh", "-c", command], capture_output=True, text=True, env=env, timeout=timeout ) From 49c65cbd5dc62f4f8c8877a042d9c0f53955fded Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 31 Oct 2026 23:24:27 +0100 Subject: [PATCH 29/50] feat: LocalAccountsMixin, personal accounts added and locked again (NetOrk/netork#1793) get_local_accounts(names) reads a person's account back: groups, lock, sudo, key files. add_local_account() creates or reopens one with the person's key, optional password hash and optional sudo rule; lock_local_account() takes the keys and sudo away and refuses every login, keeping the home. Every command root runs is one fixed command, so a sudoers allowlist can name it. The key files are read and written as the person (sudo -n -u), never as root, since a link in a home the person owns may point anywhere. The sudo rule is a drop-in of its own, checked with visudo before and sudo -l after; a rule elsewhere (cloud-init's) is reported, not edited. The login user's own account and system accounts are refused. A driver supplies creating, opening, locking, reading the lock and leaving a group. --- CHANGELOG.md | 9 + napalm_device_types/__init__.py | 9 + napalm_device_types/local_accounts.py | 565 +++++++++++++++++++++++ tests/test_local_accounts.py | 641 ++++++++++++++++++++++++++ 4 files changed, 1224 insertions(+) create mode 100644 napalm_device_types/local_accounts.py create mode 100644 tests/test_local_accounts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b04273..3c0d4d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,15 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `LocalAccountsMixin` (`napalm_device_types.local_accounts`): personal login accounts + on a host. `get_local_accounts(names)` reads them back (groups, lock, sudo, key + files), `add_local_account(name, public_key, password_hash=, admin=)` creates or + reopens one, `lock_local_account(name)` takes its keys and sudo away and refuses + every login while keeping its home. Every command root runs is one fixed command, + the key files are read and written as the account itself, and the sudo rule is a + drop-in of its own, checked with `visudo` and `sudo -l`. A driver supplies the + commands that create, open and lock an account and take it out of a group + (NetOrk/netork#1793). - `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how old an entry is. `0.0` means "confirmed just now" and is no stand-in for it diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index b61a8ae..35e95ed 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -48,6 +48,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.interface_filter.InterfaceFilterMixin` * :class:`~napalm_device_types.kernel.KernelFactsMixin` * :class:`~napalm_device_types.listening.ListeningSocketsMixin` +* :class:`~napalm_device_types.local_accounts.LocalAccountsMixin` * :class:`~napalm_device_types.host_firewall.HostFirewallMixin` * :class:`~napalm_device_types.mac_acl.MacAclMixin` * :class:`~napalm_device_types.nat_vpn.NatVpnMixin` @@ -123,6 +124,11 @@ from napalm_device_types.listening import ( ListeningSocketsUnavailable, parse_listening_sockets, ) +from napalm_device_types.local_accounts import ( + LocalAccountChangeDict, + LocalAccountDict, + LocalAccountsMixin, +) from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver @@ -317,6 +323,9 @@ __all__ = [ "ListeningSocketsMixin", "ListeningSocketsUnavailable", "parse_listening_sockets", + "LocalAccountsMixin", + "LocalAccountDict", + "LocalAccountChangeDict", "PhoneDriver", "PingSweepMixin", "PortSpec", diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py new file mode 100644 index 0000000..553eea8 --- /dev/null +++ b/napalm_device_types/local_accounts.py @@ -0,0 +1,565 @@ +"""Personal login accounts on a host: reading them, adding one, locking one again. + +A person who is to work on a host gets an account of their own: they log in +with their public key and, where they are to administer the host, may use +sudo with their own password. When they leave, the account is locked, not +deleted: their key goes, sudo goes, the login is refused, and the home stays. + +:class:`LocalAccountsMixin` holds what is the same on every POSIX host -- the +checks, reading an account back, the order of the steps, the key file and the +sudo rule. A driver supplies the commands that differ between operating +systems: creating, opening and locking an account, reading its lock, and +taking it out of a group. + +**Every command root runs is one fixed command**, with no shell syntax of its +own -- ``useradd ...``, ``install -m 440 ...``, ``rm -f -- `` -- so a +sudoers allowlist of commands can name each one. What goes into a file is +staged first, without privilege; root never reads stdin, where a sudo that +does not prompt would leave the password. + +**Nothing root runs follows the person's files.** The key files live in a home +the person owns, where a symbolic link may point anywhere, so they are read and +written as that person (``sudo -n -u ``). Only the existence of a key +file is asked as root. + +**The sudo rule is a file of its own** in the drop-in directory sudo includes, +`` ALL=(ALL) ALL``: the person's own password, as on a new VM. A group +alone is not enough, as some distributions grant ``%wheel`` and others +``%sudo``. ``visudo`` checks the rule before it is installed, and ``sudo -l`` +afterwards shows whether it took effect. A rule elsewhere -- cloud-init's +``90-cloud-init-users`` -- is not edited; it is reported. + +**The login user's own account and system accounts are never touched**: +locking the first would lock the caller out. +""" + +from __future__ import annotations + +import re +import shlex +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, ClassVar + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepDict, FixStepOutcomeDict + +if TYPE_CHECKING: # pragma: no cover - for type checkers only + from napalm_device_types.channel import CommandResult + +#: A login name every POSIX host takes as it is: lowercase, a letter or an +#: underscore first, at most 32 characters. It is also safe in a root command. +ACCOUNT_NAME_RE = re.compile(r"[a-z_][a-z0-9_.-]{0,31}") + +#: The crypt(3) prefixes of each password scheme a driver may take. +PASSWORD_SCHEMES: dict[str, tuple[str, ...]] = { + "sha512": ("$6$",), + "bcrypt": ("$2b$", "$2a$", "$2y$"), +} +_CRYPT_RE = re.compile(r"\$[0-9a-z]+\$[./0-9A-Za-z$]+") + +#: The public key types OpenSSH takes in a key file. +_KEY_TYPE_RE = re.compile( + r"(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp(256|384|521)" + r"|sk-ssh-ed25519@openssh\.com|sk-ecdsa-sha2-nistp256@openssh\.com)" +) +_KEY_BLOB_RE = re.compile(r"[A-Za-z0-9+/]+={0,3}") + +#: Before the rule's file name; after cloud-init's ``90-`` it would win, so before it. +SUDOERS_FILE_PREFIX = "80-account-" + +#: A line of ``sudo -l`` that lets the account run every command as root. +_RUN_ALL_RE = re.compile(r"^\s*\(ALL(\s*:\s*ALL)?\)\s+([A-Z_]+:\s*)*ALL\s*$", re.MULTILINE) + +#: Stages stdin as a file anyone may read -- the person reads the key file +#: from it -- and prints its path. The content is a public key or a sudo rule. +_STAGE = 't=$(mktemp) && chmod 644 "$t" && cat > "$t" && printf %s "$t"' + + +class LocalAccountDict(TypedDict): + """A person's account as the host has it (``get_local_accounts``).""" + + name: str + uid: int + home: str + shell: str + #: Primary group first. + groups: list[str] + #: No login, with a key or a password. + locked: bool + #: ``sudo -l`` lets it run every command as root. + admin: bool + #: The lines of its key files; None when they could not be read (a locked + #: account cannot run a command any more). + authorized_keys: list[str] | None + + +class LocalAccountChangeDict(FixResultDict): + """The result of ``add_local_account`` and ``lock_local_account``.""" + + #: The account read back after the change; None when it is not there or + #: could not be read. + account: LocalAccountDict | None + + +def sudoers_rule(name: str) -> str: + """The rule that lets *name* run everything as root, with its own password.""" + return f"{name} ALL=(ALL) ALL\n" + + +def sudoers_rule_path(directory: str, name: str) -> str: + """Where *name*'s rule lives in *directory*. + + sudo skips a file whose name holds a dot, so a dot becomes ``_2e``, and an + underscore ``__``: no two names share a file. + """ + escaped = name.replace("_", "__").replace(".", "_2e") + return f"{directory}/{SUDOERS_FILE_PREFIX}{escaped}" + + +def authorized_keys_paths(home: str) -> tuple[str, str]: + """The key files sshd reads in *home* unless configured otherwise.""" + return f"{home}/.ssh/authorized_keys", f"{home}/.ssh/authorized_keys2" + + +def check_account_name(name: str) -> None: + """:raises ValueError: when *name* is no login name every host takes.""" + if not ACCOUNT_NAME_RE.fullmatch(name): + raise ValueError(f"not an account name: {name!r}") + + +def _key_blob(line: str) -> str | None: + """The base64 part of a key file line, options and comment aside.""" + words = line.split() + for i, word in enumerate(words[:-1]): + if _KEY_TYPE_RE.fullmatch(word): + return words[i + 1] + return None + + +def check_public_key(key: str) -> None: + """:raises ValueError: when *key* is not one OpenSSH public key on one line.""" + words = key.split() + if ( + "\n" in key.strip() + or len(words) < 2 + or not _KEY_TYPE_RE.fullmatch(words[0]) + or not _KEY_BLOB_RE.fullmatch(words[1]) + ): + raise ValueError("not a public key: expected ' [comment]' on one line") + + +def _command(argv: Sequence[str]) -> str: + return shlex.join(argv) + + +def _as(name: str, *argv: str) -> str: + """*argv* run as *name*, from root: nothing it does reaches beyond that account.""" + return _command(["sudo", "-n", "-u", name, *argv]) + + +def _said(result: CommandResult) -> str: + return (result.stderr or result.stdout).strip() or f"exit {result.exit_code}" + + +class LocalAccountsMixin: + """Reads, adds and locks personal accounts; a driver supplies the OS's commands. + + Mixed in by the drivers that can, so ``callable(getattr(driver, + "add_local_account", None))`` says whether a driver can. + """ + + #: The login shell of a new account. + ACCOUNT_SHELL: ClassVar[str] = "/bin/sh" + #: The password scheme the host's accounts take, a key of :data:`PASSWORD_SCHEMES`. + ACCOUNT_PASSWORD_SCHEME: ClassVar[str] = "sha512" + #: Groups that make an account an administrator; a person without sudo leaves them. + ACCOUNT_ADMIN_GROUPS: ClassVar[tuple[str, ...]] = ("wheel",) + #: The lowest uid of a person's account; below are the system's. + ACCOUNT_MIN_UID: ClassVar[int] = 1000 + #: The drop-in directory sudo includes. + SUDOERS_DIR: ClassVar[str] = "/etc/sudoers.d" + #: The group of root's files. + ROOT_GROUP: ClassVar[str] = "root" + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: ... + + # -- what a driver supplies ------------------------------------------------------ + + def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: + """The commands that create *name* with a home and :attr:`ACCOUNT_SHELL`. + + Without *password_hash* the account takes no password, only a key. + """ + raise NotImplementedError + + def _account_open( + self, name: str, password_hash: str | None, *, locked: bool + ) -> list[list[str]]: + """The commands that let *name* log in again, with *password_hash* if given.""" + raise NotImplementedError + + def _account_lock(self, name: str, *, locked: bool) -> list[list[str]]: + """The commands that refuse *name* every login, a key's too; none when *locked*.""" + raise NotImplementedError + + def _account_locked(self, name: str, passwd: list[str]) -> bool: + """Whether *name* is locked; *passwd* is its ``getent passwd`` line, split. + + :raises RuntimeError: when that cannot be read. + """ + raise NotImplementedError + + def _account_leave_group(self, name: str, group: str, groups: list[str]) -> list[str]: + """The command that takes *name* out of *group*; *groups* are all of its own.""" + raise NotImplementedError + + # -- reading --------------------------------------------------------------------- + + def get_local_accounts(self, names: Iterable[str]) -> list[LocalAccountDict]: + """The accounts of *names* the host has, in that order; a missing one is left out. + + :raises ValueError: for a name that is no account name. + :raises RuntimeError: when an account cannot be read. + """ + wanted = list(names) + for name in wanted: + check_account_name(name) + found = (self._accounts_read(name) for name in wanted) + return [account for account in found if account is not None] + + def _accounts_passwd(self, name: str) -> list[str] | None: + result = self.run_command(_command(["getent", "passwd", name]), timeout=30) + if result.exit_code == 2: + return None + fields = result.stdout.strip().split(":") + if result.exit_code != 0 or len(fields) < 7 or fields[0] != name: + raise RuntimeError(f"cannot read the account {name}: {_said(result)}") + return fields + + def _accounts_read(self, name: str) -> LocalAccountDict | None: + passwd = self._accounts_passwd(name) + if passwd is None: + return None + groups = self.run_command(_command(["id", "-Gn", name]), timeout=30) + if groups.exit_code != 0: + raise RuntimeError(f"cannot read the groups of {name}: {_said(groups)}") + home = passwd[5] + return { + "name": name, + "uid": int(passwd[2]), + "home": home, + "shell": passwd[6], + "groups": groups.stdout.split(), + "locked": self._account_locked(name, passwd), + "admin": self._accounts_admin(name), + "authorized_keys": self._accounts_keys(name, home), + } + + def _accounts_admin(self, name: str) -> bool: + result = self.run_command(_command(["sudo", "-l", "-U", name]), privileged=True, timeout=30) + if result.exit_code != 0: + raise RuntimeError(f"cannot ask sudo about {name}: {_said(result)}") + return bool(_RUN_ALL_RE.search(result.stdout)) + + def _accounts_keys(self, name: str, home: str) -> list[str] | None: + """The lines of *name*'s key files, read as *name*; None when it cannot.""" + lines: list[str] = [] + for path in authorized_keys_paths(home): + there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) + if there.exit_code != 0: + continue + result = self.run_command(_as(name, "cat", "--", path), privileged=True, timeout=30) + if result.exit_code != 0: + return None + lines += [ln for ln in result.stdout.splitlines() if ln.strip()] + return lines + + # -- changing ---------------------------------------------------------------------- + + def add_local_account( + self, + name: str, + public_key: str, + *, + password_hash: str | None = None, + admin: bool = False, + ) -> LocalAccountChangeDict: + """Give *name* an account that logs in with *public_key*, or open it again. + + An account that is there is opened again and keeps its other keys; a + new one gets a home. *password_hash* (crypt(3), in + :attr:`ACCOUNT_PASSWORD_SCHEME`) becomes its password; *admin* gives it + the sudo rule, which asks for that password -- so an administrator + needs one. Without *admin*, the rule and :attr:`ACCOUNT_ADMIN_GROUPS` + are taken away. + + Steps: ``account``, then ``sudo`` and ``keys``, each of which runs when + the other failed. + + :raises ValueError: for a name, key or hash no host takes, the login + user's own account, or a system account. + """ + check_account_name(name) + check_public_key(public_key) + if password_hash is not None: + self._accounts_check_hash(password_hash) + existing = self._accounts_changeable(name) + first = run_fix_steps( + [("account", lambda: self._accounts_open_or_create(name, existing, password_hash))] + ) + parts = [first] + if first["success"]: + parts += [ + run_fix_steps([("sudo", lambda: self._accounts_set_sudo(name, admin=admin))]), + run_fix_steps([("keys", lambda: self._accounts_add_key(name, public_key))]), + ] + return self._accounts_result(name, parts) + + def lock_local_account(self, name: str) -> LocalAccountChangeDict: + """Lock *name*: its keys go, every login is refused, sudo goes; its home stays. + + Steps ``keys``, ``lock`` and ``sudo``, each run when one before failed: + the lock matters most. A sudo rule outside the drop-in directory is + reported, not edited -- with the account locked it can no longer be used. + + :raises ValueError: for a name no host takes, the login user's own + account, or a system account. + """ + check_account_name(name) + if self._accounts_changeable(name) is None: + step: FixStepDict = {"step": "account", "ok": False, "output": f"no account {name}"} + return {**_joined([step]), "account": None} + parts = [ + run_fix_steps([("keys", lambda: self._accounts_remove_keys(name))]), + run_fix_steps([("lock", lambda: self._accounts_lock(name))]), + run_fix_steps( + [("sudo", lambda: self._accounts_set_sudo(name, admin=False, locked=True))] + ), + ] + return self._accounts_result(name, parts) + + def _accounts_check_hash(self, password_hash: str) -> None: + prefixes = PASSWORD_SCHEMES[self.ACCOUNT_PASSWORD_SCHEME] + if not _CRYPT_RE.fullmatch(password_hash) or not password_hash.startswith(prefixes): + raise ValueError(f"not a {self.ACCOUNT_PASSWORD_SCHEME} password hash") + + def _accounts_changeable(self, name: str) -> list[str] | None: + """*name*'s passwd line, None when there is none. + + :raises ValueError: for the login user's own account or a system account. + :raises RuntimeError: when the login user cannot be told. + """ + login = self.run_command("id -un", timeout=30).stdout.strip() + if not login: + raise RuntimeError("cannot tell which account this driver logs in with") + if name == login: + raise ValueError(f"{name} is the account this driver logs in with") + passwd = self._accounts_passwd(name) + if passwd is not None and int(passwd[2]) < self.ACCOUNT_MIN_UID: + raise ValueError(f"{name} is a system account (uid {passwd[2]})") + return passwd + + def _accounts_result(self, name: str, parts: list[FixResultDict]) -> LocalAccountChangeDict: + steps = [s for part in parts for s in part["steps"]] + result = _joined(steps) + account: LocalAccountDict | None = None + try: + account = self._accounts_read(name) + except RuntimeError as exc: + result["output"] += f"\n[read back] {exc}" + return {**result, "account": account} + + def _accounts_root(self, argv: Sequence[str]) -> CommandResult: + return self.run_command(_command(argv), privileged=True, timeout=60) + + def _accounts_run(self, commands: list[list[str]]) -> CommandResult | None: + """Run *commands* as root in order; the first that failed, or None.""" + for argv in commands: + result = self._accounts_root(argv) + if result.exit_code != 0: + return result + return None + + def _accounts_open_or_create( + self, name: str, passwd: list[str] | None, password_hash: str | None + ) -> FixStepOutcomeDict: + if passwd is None: + failed = self._accounts_run(self._account_create(name, password_hash)) + done = f"account {name} created" + else: + locked = self._account_locked(name, passwd) + failed = self._accounts_run(self._account_open(name, password_hash, locked=locked)) + done = f"account {name} opened again" if locked else f"account {name} is there" + if failed is not None: + return {"ok": False, "output": _said(failed)} + return {"ok": True, "output": done + (", password set" if password_hash else "")} + + def _accounts_lock(self, name: str) -> FixStepOutcomeDict: + passwd = self._accounts_passwd(name) + if passwd is None: + return {"ok": False, "output": f"no account {name}"} + locked = self._account_locked(name, passwd) + failed = self._accounts_run(self._account_lock(name, locked=locked)) + if failed is not None: + return {"ok": False, "output": _said(failed)} + if not self._account_locked(name, passwd): + return {"ok": False, "output": f"{name} is still not locked"} + return {"ok": True, "output": f"{name} was locked already" if locked else f"{name} locked"} + + # -- sudo -------------------------------------------------------------------------- + + def _accounts_set_sudo( + self, name: str, *, admin: bool, locked: bool = False + ) -> FixStepOutcomeDict: + path = sudoers_rule_path(self.SUDOERS_DIR, name) + if admin: + return self._accounts_grant_sudo(name, path) + said: list[str] = [] + removed = self._accounts_root(["rm", "-f", "--", path]) + if removed.exit_code != 0: + return {"ok": False, "output": f"{path}: {_said(removed)}"} + groups = self.run_command(_command(["id", "-Gn", name]), timeout=30).stdout.split() + for group in self.ACCOUNT_ADMIN_GROUPS: + if group not in groups[1:]: + continue + left = self._accounts_root(self._account_leave_group(name, group, groups)) + if left.exit_code != 0: + return {"ok": False, "output": f"cannot take {name} out of {group}: {_said(left)}"} + groups.remove(group) + said.append(f"{name} left {group}") + if self._accounts_admin(name): + rest = f"another rule than {path} still lets {name} run everything as root" + if locked: + return {"ok": True, "output": f"{rest}; the account is locked, so it cannot use it"} + return {"ok": False, "output": rest} + said.append(f"{name} has no sudo") + return {"ok": True, "output": "\n".join(said)} + + def _accounts_grant_sudo(self, name: str, path: str) -> FixStepOutcomeDict: + staged = self._accounts_stage(sudoers_rule(name)) + try: + checked = self._accounts_root(["visudo", "-cf", staged]) + if checked.exit_code != 0: + return {"ok": False, "output": f"visudo refuses the rule: {_said(checked)}"} + installed = self._accounts_root( + ["install", "-m", "440", "-o", "root", "-g", self.ROOT_GROUP, "--", staged, path] + ) + if installed.exit_code != 0: + return {"ok": False, "output": f"{path}: {_said(installed)}"} + finally: + self._accounts_unstage(staged) + if not self._accounts_admin(name): + return { + "ok": False, + "output": f"{path} is written, but sudo does not read it: " + f"does sudoers include {self.SUDOERS_DIR}?", + } + return {"ok": True, "output": f"{name} may use sudo with its password ({path})"} + + # -- keys -------------------------------------------------------------------------- + + def _accounts_stage(self, content: str) -> str: + """*content* in a fresh file of the login user's; its path. + + :raises RuntimeError: when it cannot be staged. + """ + result = self.run_command(_STAGE, timeout=30, stdin=content.encode()) + path = result.stdout.strip().splitlines()[-1:] or [""] + if result.exit_code != 0 or not path[0].startswith("/"): + raise RuntimeError(f"cannot stage a file: {_said(result)}") + return path[0] + + def _accounts_unstage(self, path: str) -> None: + self.run_command(_command(["rm", "-f", "--", path]), timeout=30) + + def _accounts_add_key(self, name: str, public_key: str) -> FixStepOutcomeDict: + passwd = self._accounts_passwd(name) + if passwd is None: + return {"ok": False, "output": f"no account {name}"} + home = passwd[5] + path = authorized_keys_paths(home)[0] + current = self._accounts_keys(name, home) + if current is None: + return {"ok": False, "output": f"cannot read the key files of {name}"} + blob = _key_blob(public_key) + if any(_key_blob(line) == blob for line in current): + return {"ok": True, "output": f"the key is already in {path}"} + mine = self._accounts_key_file(name, path) + if mine is None: + return {"ok": False, "output": f"cannot read {path}"} + staged = self._accounts_stage("".join(f"{line}\n" for line in [*mine, public_key.strip()])) + try: + failed = self._accounts_run_as( + name, + [ + ["mkdir", "-p", "-m", "700", "--", f"{home}/.ssh"], + ["install", "-m", "600", "--", staged, path], + ], + ) + finally: + self._accounts_unstage(staged) + if failed is not None: + return {"ok": False, "output": f"{path}: {_said(failed)}"} + return {"ok": True, "output": f"key added to {path}"} + + def _accounts_key_file(self, name: str, path: str) -> list[str] | None: + there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) + if there.exit_code != 0: + return [] + result = self.run_command(_as(name, "cat", "--", path), privileged=True, timeout=30) + if result.exit_code != 0: + return None + return [line for line in result.stdout.splitlines() if line.strip()] + + def _accounts_run_as(self, name: str, commands: list[list[str]]) -> CommandResult | None: + for argv in commands: + result = self.run_command(_as(name, *argv), privileged=True, timeout=60) + if result.exit_code != 0: + return result + return None + + def _accounts_remove_keys(self, name: str) -> FixStepOutcomeDict: + passwd = self._accounts_passwd(name) + if passwd is None: + return {"ok": False, "output": f"no account {name}"} + paths = authorized_keys_paths(passwd[5]) + failed = self._accounts_run_as(name, [["rm", "-f", "--", *paths]]) + if failed is not None: + return {"ok": False, "output": f"cannot remove the keys of {name}: {_said(failed)}"} + if self._accounts_keys(name, passwd[5]): + return {"ok": False, "output": f"{name} still has keys"} + return {"ok": True, "output": f"keys of {name} removed"} + + +def _joined(steps: list[FixStepDict]) -> FixResultDict: + return { + "success": bool(steps) and all(s["ok"] for s in steps), + "steps": steps, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in steps), + } + + +__all__ = [ + "ACCOUNT_NAME_RE", + "PASSWORD_SCHEMES", + "SUDOERS_FILE_PREFIX", + "LocalAccountChangeDict", + "LocalAccountDict", + "LocalAccountsMixin", + "authorized_keys_paths", + "check_account_name", + "check_public_key", + "sudoers_rule", + "sudoers_rule_path", +] diff --git a/tests/test_local_accounts.py b/tests/test_local_accounts.py new file mode 100644 index 0000000..4c36b7a --- /dev/null +++ b/tests/test_local_accounts.py @@ -0,0 +1,641 @@ +"""Personal accounts on a host: reading them, adding one, locking one again. + +The host is a fake that keeps accounts, groups, sudo rules and key files in +memory and answers the fixed commands the mixin and a driver send. A driver's +own commands (create, open, lock, leave a group) are stand-ins here; the real +ones are tested in the drivers. +""" + +from __future__ import annotations + +import shlex +from dataclasses import dataclass, field + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.local_accounts import ( + LocalAccountsMixin, + authorized_keys_paths, + sudoers_rule, + sudoers_rule_path, +) + +ALICE_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAlice alice@laptop" +OWN_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQOwn own@desktop" +HASH = "$6$salt$" + "x" * 86 +#: Shell syntax a command for root must not hold: each is one fixed command. +SHELL_SYNTAX = (";", "&&", "||", "|", ">", "<", "$(", "`") + + +@dataclass +class _Account: + uid: int + home: str + shell: str = "/bin/sh" + groups: list[str] = field(default_factory=list) + locked: bool = False + password: str | None = None + + +class _Host: + """What the fake host holds, and the commands it ran.""" + + def __init__(self) -> None: + self.login = "netadmin" + self.accounts: dict[str, _Account] = { + "netadmin": _Account(1000, "/home/netadmin", groups=["netadmin", "wheel"]), + "daemon": _Account(1, "/usr/sbin"), + } + #: Path -> content, for the sudoers drop-ins and the key files. + self.files: dict[str, str] = {} + #: Users a rule outside the drop-in directory lets run everything. + self.other_rules: set[str] = set() + #: Whether sudo reads the drop-in directory at all. + self.includes_dropins = True + #: Commands that fail, by their first words ("as " for one run as + #: the person); and commands that succeed without doing anything. + self.failing: set[str] = set() + self.ignored: set[str] = set() + self.ran: list[tuple[str, bool]] = [] + self.staged = 0 + + def is_admin(self, name: str) -> bool: + rule = self.files.get(sudoers_rule_path(_Driver.SUDOERS_DIR, name)) + return name in self.other_rules or (self.includes_dropins and rule is not None) + + def run(self, command: str, privileged: bool, stdin: bytes | None) -> CommandResult: + self.ran.append((command, privileged)) + if stdin is not None: + assert not privileged, "a privileged command takes no stdin" + return self._stage(command, stdin) + words = shlex.split(command) + if " ".join(words[:2]) in self.failing or words[0] in self.failing: + return CommandResult("", f"{words[0]}: failed", 1) + if words[:3] == ["sudo", "-n", "-u"]: + assert privileged, "only root changes to another user" + return self._as_user(words[3], words[4:]) + handler = getattr(self, "_" + words[0].replace("-", "_"), None) + assert handler is not None, f"unexpected command: {command}" + return handler(words[1:], privileged) + + # -- staging, as the login user --------------------------------------------- + + def _stage(self, command: str, stdin: bytes) -> CommandResult: + assert "mktemp" in command + if "mktemp" in self.failing: + return CommandResult("", "mktemp: No space left on device", 1) + self.staged += 1 + path = f"/tmp/tmp.{self.staged}" + self.files[path] = stdin.decode() + return CommandResult(path, "", 0) + + def _rm(self, args: list[str], privileged: bool) -> CommandResult: + for path in args[args.index("--") + 1 :]: + if path.startswith("/tmp/"): + assert not privileged, "the staged file is the login user's" + self.files.pop(path, None) + return CommandResult("", "", 0) + + # -- the account database ------------------------------------------------------- + + def _getent(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "passwd" and not privileged + a = self.accounts.get(args[1]) + if a is None: + return CommandResult("", "", 2) + return CommandResult(f"{args[1]}:*:{a.uid}:{a.uid}::{a.home}:{a.shell}\n", "", 0) + + def _id(self, args: list[str], privileged: bool) -> CommandResult: + if args == ["-un"]: + return CommandResult(f"{self.login}\n", "", 0) + assert args[0] == "-Gn" and not privileged + a = self.accounts.get(args[1]) + if a is None: + return CommandResult("", f"id: {args[1]}: no such user", 1) + return CommandResult(" ".join(a.groups) + "\n", "", 0) + + def _sudo(self, args: list[str], privileged: bool) -> CommandResult: + assert args[:2] == ["-l", "-U"] and privileged + name = args[2] + if self.is_admin(name): + out = f"User {name} may run the following commands on host:\n (ALL) ALL\n" + else: + out = f"User {name} is not allowed to run sudo on host.\n" + return CommandResult(out, "", 0) + + def _visudo(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "-cf" and privileged + content = self.files[args[1]] + ok = content.endswith(" ALL=(ALL) ALL\n") and "\n" not in content[:-1] + return CommandResult("parsed OK" if ok else "syntax error", "", 0 if ok else 1) + + def _install(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + source, target = args[args.index("--") + 1 :] + assert target.startswith(_Driver.SUDOERS_DIR + "/"), "root installs only sudo rules" + assert args[:6] == ["-m", "440", "-o", "root", "-g", _Driver.ROOT_GROUP] + self.files[target] = self.files[source] + return CommandResult("", "", 0) + + def _test(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "-e" + return CommandResult("", "", 0 if args[1] in self.files else 1) + + # -- the person's own files, as that person ------------------------------------ + + def _as_user(self, name: str, words: list[str]) -> CommandResult: + a = self.accounts[name] + if a.locked: + return CommandResult("", "sudo: account expired", 1) + rest = words[words.index("--") + 1 :] if "--" in words else words[1:] + if f"as {words[0]}" in self.failing: + return CommandResult("", f"{words[0]}: Permission denied", 1) + if f"as {words[0]}" in self.ignored: + return CommandResult("", "", 0) + assert all(p.startswith(a.home + "/") or p.startswith("/tmp/") for p in rest) + if words[0] == "cat": + return CommandResult(self.files.get(rest[0], ""), "", 0) + if words[0] == "mkdir": + return CommandResult("", "", 0) + if words[0] == "install": + assert words[1:3] == ["-m", "600"] + self.files[rest[1]] = self.files[rest[0]] + return CommandResult("", "", 0) + if words[0] == "rm": + for path in rest: + self.files.pop(path, None) + return CommandResult("", "", 0) + raise AssertionError(f"unexpected command as {name}: {words}") + + # -- the driver's own commands (stand-ins) ------------------------------------- + + def _mkuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + name, password = args[0], (args[1] if len(args) > 1 else None) + uid = 1001 + len(self.accounts) + self.accounts[name] = _Account(uid, f"/home/{name}", groups=[name], password=password) + return CommandResult("", "", 0) + + def _openuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + a = self.accounts[args[0]] + a.locked = False + if len(args) > 1: + a.password = args[1] + return CommandResult("", "", 0) + + def _lockuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + self.accounts[args[0]].locked = "lockuser" not in self.ignored + return CommandResult("", "", 0) + + def _islocked(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + return CommandResult("yes\n" if self.accounts[args[0]].locked else "no\n", "", 0) + + def _leavegroup(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + self.accounts[args[0]].groups.remove(args[1]) + return CommandResult("", "", 0) + + +class _Driver(LocalAccountsMixin): + ACCOUNT_SHELL = "/bin/sh" + ACCOUNT_ADMIN_GROUPS = ("wheel", "sudo") + SUDOERS_DIR = "/etc/sudoers.d" + ROOT_GROUP = "wheel" + + def __init__(self, host: _Host) -> None: + self.host = host + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: + return self.host.run(command, privileged, stdin) + + def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: + return [["mkuser", name, *([password_hash] if password_hash else [])]] + + def _account_open( + self, name: str, password_hash: str | None, *, locked: bool + ) -> list[list[str]]: + return [["openuser", name, *([password_hash] if password_hash else [])]] + + def _account_lock(self, name: str, *, locked: bool) -> list[list[str]]: + return [] if locked else [["lockuser", name]] + + def _account_locked(self, name: str, passwd: list[str]) -> bool: + result = self.run_command(shlex.join(["islocked", name]), privileged=True) + return result.stdout.strip() == "yes" + + def _account_leave_group(self, name: str, group: str, groups: list[str]) -> list[str]: + return ["leavegroup", name, group] + + +@pytest.fixture +def host() -> _Host: + return _Host() + + +@pytest.fixture +def driver(host: _Host) -> _Driver: + return _Driver(host) + + +def _root_commands(host: _Host) -> list[str]: + return [c for c, privileged in host.ran if privileged] + + +def _add_alice(host: _Host, *, locked: bool = False, keys: str = "") -> None: + host.accounts["alice"] = _Account( + 1001, "/home/alice", groups=["alice", "wheel"], locked=locked, password=HASH + ) + if keys: + host.files["/home/alice/.ssh/authorized_keys"] = keys + + +# -- the helpers --------------------------------------------------------------------- + + +def test_the_sudo_rule_names_the_account_and_asks_for_its_password() -> None: + assert sudoers_rule("alice") == "alice ALL=(ALL) ALL\n" + + +def test_the_rule_file_has_no_dot_sudo_would_skip() -> None: + # sudo ignores a file in an included directory whose name holds a dot. + path = sudoers_rule_path("/etc/sudoers.d", "a.lice") + assert path.startswith("/etc/sudoers.d/") + assert "." not in path.rsplit("/", 1)[1] + assert sudoers_rule_path("/etc/sudoers.d", "a.lice") != sudoers_rule_path( + "/etc/sudoers.d", "a_lice" + ) + + +def test_both_key_files_sshd_reads_by_default() -> None: + assert authorized_keys_paths("/home/alice") == ( + "/home/alice/.ssh/authorized_keys", + "/home/alice/.ssh/authorized_keys2", + ) + + +# -- reading -------------------------------------------------------------------------- + + +def test_an_account_is_read_with_its_groups_lock_sudo_and_keys( + host: _Host, driver: _Driver +) -> None: + _add_alice(host, keys=ALICE_KEY + "\n" + OWN_KEY + "\n") + host.other_rules.add("alice") + + [account] = driver.get_local_accounts(["alice"]) + + assert account == { + "name": "alice", + "uid": 1001, + "home": "/home/alice", + "shell": "/bin/sh", + "groups": ["alice", "wheel"], + "locked": False, + "admin": True, + "authorized_keys": [ALICE_KEY, OWN_KEY], + } + + +def test_an_account_that_is_not_there_is_left_out(driver: _Driver) -> None: + assert driver.get_local_accounts(["bob"]) == [] + + +def test_a_locked_accounts_keys_that_cannot_be_read_are_unknown( + host: _Host, driver: _Driver +) -> None: + _add_alice(host, locked=True, keys=ALICE_KEY + "\n") + + [account] = driver.get_local_accounts(["alice"]) + + assert account["locked"] is True + assert account["authorized_keys"] is None + + +def test_an_account_without_a_key_file_has_no_keys(host: _Host, driver: _Driver) -> None: + _add_alice(host) + + [account] = driver.get_local_accounts(["alice"]) + + assert account["authorized_keys"] == [] + + +def test_a_failing_read_raises_rather_than_report_nothing(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.failing.add("sudo -l") + + with pytest.raises(RuntimeError, match="alice"): + driver.get_local_accounts(["alice"]) + + +@pytest.mark.parametrize("name", ["", "Alice", "1alice", "al ice", "a;b", "-alice", "a" * 33]) +def test_a_name_no_guest_takes_is_refused(driver: _Driver, name: str) -> None: + with pytest.raises(ValueError, match="account name"): + driver.get_local_accounts([name]) + + +# -- adding --------------------------------------------------------------------------- + + +def test_a_new_account_gets_the_key_and_the_sudo_rule(host: _Host, driver: _Driver) -> None: + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["account", "sudo", "keys"] + assert host.accounts["alice"].password == HASH + assert host.files["/home/alice/.ssh/authorized_keys"] == ALICE_KEY + "\n" + assert host.files["/etc/sudoers.d/80-account-alice"] == "alice ALL=(ALL) ALL\n" + account = result["account"] + assert account is not None + assert account["admin"] is True and account["authorized_keys"] == [ALICE_KEY] + + +def test_every_command_root_runs_is_one_fixed_command(host: _Host, driver: _Driver) -> None: + driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + driver.lock_local_account("alice") + + for command in _root_commands(host): + assert not any(s in command for s in SHELL_SYNTAX), command + + +def test_the_staged_files_are_removed_again(host: _Host, driver: _Driver) -> None: + driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert host.staged == 2 + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_an_existing_account_keeps_its_own_keys(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys=OWN_KEY + "\n") + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.files["/home/alice/.ssh/authorized_keys"] == f"{OWN_KEY}\n{ALICE_KEY}\n" + + +def test_a_line_without_a_key_is_kept_and_not_taken_for_one(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys="# the laptop\n") + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.files["/home/alice/.ssh/authorized_keys"] == f"# the laptop\n{ALICE_KEY}\n" + + +def test_a_key_already_there_is_not_written_again(host: _Host, driver: _Driver) -> None: + # The same key with options in front and another comment is the same key. + _add_alice(host, keys=f'from="10.0.0.0/8" {ALICE_KEY.rsplit(" ", 1)[0]} old\n') + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.staged == 0 + assert "already" in result["steps"][-1]["output"] + + +def test_a_locked_account_is_opened_again(host: _Host, driver: _Driver) -> None: + _add_alice(host, locked=True) + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH) + + assert result["success"], result["output"] + assert host.accounts["alice"].locked is False + assert result["account"] is not None and result["account"]["locked"] is False + + +def test_without_sudo_the_rule_and_the_admin_groups_go(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.files["/etc/sudoers.d/80-account-alice"] = sudoers_rule("alice") + + result = driver.add_local_account("alice", ALICE_KEY, admin=False) + + assert result["success"], result["output"] + assert "/etc/sudoers.d/80-account-alice" not in host.files + assert host.accounts["alice"].groups == ["alice"] + + +def test_sudo_another_rule_still_grants_fails_the_step(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.other_rules.add("alice") + + result = driver.add_local_account("alice", ALICE_KEY, admin=False) + + assert not result["success"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "another rule" in sudo["output"] + # The key still went in: the steps after the account are independent. + assert host.files["/home/alice/.ssh/authorized_keys"] == ALICE_KEY + "\n" + + +def test_a_rule_sudo_does_not_read_fails_the_step(host: _Host, driver: _Driver) -> None: + host.includes_dropins = False + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "/etc/sudoers.d" in sudo["output"] + + +def test_a_rule_visudo_rejects_is_not_installed(host: _Host, driver: _Driver) -> None: + host.failing.add("visudo") + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert not result["success"] + assert "/etc/sudoers.d/80-account-alice" not in host.files + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_a_failed_create_stops_before_sudo_and_keys(host: _Host, driver: _Driver) -> None: + host.failing.add("mkuser") + + result = driver.add_local_account("alice", ALICE_KEY, admin=True) + + assert not result["success"] + assert [s["step"] for s in result["steps"]] == ["account"] + assert result["account"] is None + + +@pytest.mark.parametrize( + "key", + ["", "not a key", ALICE_KEY + "\n" + OWN_KEY, "ssh-ed25519", "ssh-ed25519 AAAA;rm"], +) +def test_a_public_key_that_is_none_is_refused(driver: _Driver, key: str) -> None: + with pytest.raises(ValueError, match="public key"): + driver.add_local_account("alice", key) + + +@pytest.mark.parametrize("password_hash", ["secret", "$6$salt$ab cd", "$2b$10$" + "x" * 53]) +def test_a_password_hash_the_guest_does_not_take_is_refused( + driver: _Driver, password_hash: str +) -> None: + # This driver takes SHA-512 crypt: neither plain text nor bcrypt. + with pytest.raises(ValueError, match="password hash"): + driver.add_local_account("alice", ALICE_KEY, password_hash=password_hash) + + +def test_the_login_users_own_account_is_never_touched(host: _Host, driver: _Driver) -> None: + for change in ( + lambda: driver.add_local_account("netadmin", ALICE_KEY), + lambda: driver.lock_local_account("netadmin"), + ): + with pytest.raises(ValueError, match="logs in"): + change() + assert _root_commands(host) == [] + + +def test_nothing_changes_when_the_login_user_cannot_be_told(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.login = "" + + with pytest.raises(RuntimeError, match="logs in"): + driver.lock_local_account("alice") + assert _root_commands(host) == [] + + +def test_a_system_account_is_never_touched(host: _Host, driver: _Driver) -> None: + with pytest.raises(ValueError, match="system account"): + driver.lock_local_account("daemon") + assert _root_commands(host) == [] + + +# -- locking -------------------------------------------------------------------------- + + +def test_locking_takes_keys_sudo_and_login_away(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys=ALICE_KEY + "\n" + OWN_KEY + "\n") + host.files["/home/alice/.ssh/authorized_keys2"] = OWN_KEY + "\n" + host.files["/etc/sudoers.d/80-account-alice"] = sudoers_rule("alice") + + result = driver.lock_local_account("alice") + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["keys", "lock", "sudo"] + a = host.accounts["alice"] + assert a.locked and a.groups == ["alice"] and a.password == HASH + assert not [p for p in host.files if p.startswith("/home/alice/")] + assert "/etc/sudoers.d/80-account-alice" not in host.files + account = result["account"] + assert account is not None and account["locked"] and not account["admin"] + + +def test_a_rule_left_elsewhere_is_named_but_harmless_once_locked( + host: _Host, driver: _Driver +) -> None: + _add_alice(host) + host.other_rules.add("alice") + + result = driver.lock_local_account("alice") + + assert result["success"], result["output"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert "another rule" in sudo["output"] and "locked" in sudo["output"] + + +def test_the_lock_comes_even_when_the_keys_could_not_go(host: _Host, driver: _Driver) -> None: + _add_alice(host, locked=True, keys=ALICE_KEY + "\n") + + result = driver.lock_local_account("alice") + + assert not result["success"] + keys = next(s for s in result["steps"] if s["step"] == "keys") + assert not keys["ok"] + assert next(s for s in result["steps"] if s["step"] == "lock")["ok"] + assert host.accounts["alice"].locked + + +def test_locking_an_account_that_is_not_there_fails(driver: _Driver) -> None: + result = driver.lock_local_account("bob") + + assert not result["success"] + assert "no account" in result["output"] + assert result["account"] is None + + +# -- what fails is said --------------------------------------------------------------- + + +@pytest.mark.parametrize("failing", ["getent", "id -Gn"]) +def test_an_account_that_cannot_be_read_raises(host: _Host, driver: _Driver, failing: str) -> None: + _add_alice(host) + host.failing.add(failing) + + with pytest.raises(RuntimeError, match="alice"): + driver.get_local_accounts(["alice"]) + + +@pytest.mark.parametrize( + ("failing", "step", "said"), + [ + ("rm -f", "sudo", "80-account-alice"), + ("leavegroup", "sudo", "out of wheel"), + ("as cat", "keys", "cannot read"), + ("as install", "keys", "authorized_keys"), + ("mktemp", "keys", "cannot stage"), + ], +) +def test_a_step_that_fails_says_why( + host: _Host, driver: _Driver, failing: str, step: str, said: str +) -> None: + _add_alice(host, keys=OWN_KEY + "\n") + host.failing.add(failing) + + result = driver.add_local_account("alice", ALICE_KEY) + + failed = next(s for s in result["steps"] if s["step"] == step) + assert not failed["ok"] and said in failed["output"] + assert not result["success"] + + +def test_a_rule_that_cannot_be_installed_fails_the_step(host: _Host, driver: _Driver) -> None: + host.failing.add("install -m") + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "80-account-alice" in sudo["output"] + assert not [p for p in host.files if p.startswith("/tmp/")] + + +@pytest.mark.parametrize( + ("failing", "ignored", "step", "said"), + [ + ({"lockuser"}, set(), "lock", "lockuser: failed"), + (set(), {"lockuser"}, "lock", "still not locked"), + ({"as rm"}, set(), "keys", "cannot remove"), + (set(), {"as rm"}, "keys", "still has keys"), + ], +) +def test_a_lock_that_did_not_take_says_so( + host: _Host, driver: _Driver, failing: set[str], ignored: set[str], step: str, said: str +) -> None: + _add_alice(host, keys=ALICE_KEY + "\n") + host.failing |= failing + host.ignored |= ignored + + result = driver.lock_local_account("alice") + + failed = next(s for s in result["steps"] if s["step"] == step) + assert not failed["ok"] and said in failed["output"] + assert not result["success"] + + +def test_an_account_that_cannot_be_read_back_is_said(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.failing.add("sudo -l") + + result = driver.lock_local_account("alice") + + assert result["account"] is None + assert "[read back] cannot ask sudo about alice" in result["output"] From 0545d1d814498a93d14672079bb0ab4a384f07ea Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 31 Oct 2026 23:29:46 +0100 Subject: [PATCH 30/50] feat: a driver may keep the account keys elsewhere (NetOrk/netork#1793) OpenMediaVault keeps a user's keys in its own database and deploys them to /var/lib/openmediavault/ssh/authorized_keys, and creates a home only when its home directory setting is on. The key store becomes three hooks with the ~/.ssh default: _account_read_keys, _account_add_key and _account_remove_keys, each given the account's home. --- CHANGELOG.md | 5 +-- napalm_device_types/local_accounts.py | 49 ++++++++++++++++++++------- tests/test_local_accounts.py | 38 +++++++++++++++++++++ 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0d4d1..b134692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,9 @@ recorded here; their history is the git log. every login while keeping its home. Every command root runs is one fixed command, the key files are read and written as the account itself, and the sudo rule is a drop-in of its own, checked with `visudo` and `sudo -l`. A driver supplies the - commands that create, open and lock an account and take it out of a group - (NetOrk/netork#1793). + commands that create, open and lock an account and take it out of a group, and may + replace where the keys are kept (`_account_read_keys`, `_account_add_key`, + `_account_remove_keys`; default `~/.ssh/authorized_keys`) (NetOrk/netork#1793). - `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how old an entry is. `0.0` means "confirmed just now" and is no stand-in for it diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py index 553eea8..f4d23d4 100644 --- a/napalm_device_types/local_accounts.py +++ b/napalm_device_types/local_accounts.py @@ -195,6 +195,10 @@ class LocalAccountsMixin: ) -> CommandResult: ... # -- what a driver supplies ------------------------------------------------------ + # + # ``_account_*`` are the driver's: the five below it must supply, the key + # store (``_account_read_keys``, ``_account_add_key``, ``_account_remove_keys``) + # it may replace. ``_accounts_*`` are this mixin's own. def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: """The commands that create *name* with a home and :attr:`ACCOUNT_SHELL`. @@ -263,7 +267,7 @@ class LocalAccountsMixin: "groups": groups.stdout.split(), "locked": self._account_locked(name, passwd), "admin": self._accounts_admin(name), - "authorized_keys": self._accounts_keys(name, home), + "authorized_keys": self._account_read_keys(name, home), } def _accounts_admin(self, name: str) -> bool: @@ -272,8 +276,12 @@ class LocalAccountsMixin: raise RuntimeError(f"cannot ask sudo about {name}: {_said(result)}") return bool(_RUN_ALL_RE.search(result.stdout)) - def _accounts_keys(self, name: str, home: str) -> list[str] | None: - """The lines of *name*'s key files, read as *name*; None when it cannot.""" + def _account_read_keys(self, name: str, home: str) -> list[str] | None: + """The lines of *name*'s key files, read as *name*; None when it cannot. + + Every key sshd would take for *name*: a driver whose host keeps keys in + a place of its own as well reads that too. + """ lines: list[str] = [] for path in authorized_keys_paths(home): there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) @@ -483,13 +491,30 @@ class LocalAccountsMixin: def _accounts_unstage(self, path: str) -> None: self.run_command(_command(["rm", "-f", "--", path]), timeout=30) - def _accounts_add_key(self, name: str, public_key: str) -> FixStepOutcomeDict: + def _accounts_home(self, name: str) -> str | None: passwd = self._accounts_passwd(name) - if passwd is None: + return passwd[5] if passwd is not None else None + + def _accounts_add_key(self, name: str, public_key: str) -> FixStepOutcomeDict: + home = self._accounts_home(name) + if home is None: return {"ok": False, "output": f"no account {name}"} - home = passwd[5] + return self._account_add_key(name, home, public_key) + + def _accounts_remove_keys(self, name: str) -> FixStepOutcomeDict: + home = self._accounts_home(name) + if home is None: + return {"ok": False, "output": f"no account {name}"} + return self._account_remove_keys(name, home) + + def _account_add_key(self, name: str, home: str, public_key: str) -> FixStepOutcomeDict: + """Add *public_key* to *name*'s first key file, as *name*; the others stay. + + A driver whose host keeps keys elsewhere overrides this, with + :meth:`_account_read_keys` and :meth:`_account_remove_keys`. + """ path = authorized_keys_paths(home)[0] - current = self._accounts_keys(name, home) + current = self._account_read_keys(name, home) if current is None: return {"ok": False, "output": f"cannot read the key files of {name}"} blob = _key_blob(public_key) @@ -529,15 +554,13 @@ class LocalAccountsMixin: return result return None - def _accounts_remove_keys(self, name: str) -> FixStepOutcomeDict: - passwd = self._accounts_passwd(name) - if passwd is None: - return {"ok": False, "output": f"no account {name}"} - paths = authorized_keys_paths(passwd[5]) + def _account_remove_keys(self, name: str, home: str) -> FixStepOutcomeDict: + """Remove *name*'s key files, as *name*.""" + paths = authorized_keys_paths(home) failed = self._accounts_run_as(name, [["rm", "-f", "--", *paths]]) if failed is not None: return {"ok": False, "output": f"cannot remove the keys of {name}: {_said(failed)}"} - if self._accounts_keys(name, passwd[5]): + if self._account_read_keys(name, home): return {"ok": False, "output": f"{name} still has keys"} return {"ok": True, "output": f"keys of {name} removed"} diff --git a/tests/test_local_accounts.py b/tests/test_local_accounts.py index 4c36b7a..c9ed53c 100644 --- a/tests/test_local_accounts.py +++ b/tests/test_local_accounts.py @@ -20,6 +20,7 @@ from napalm_device_types.local_accounts import ( sudoers_rule, sudoers_rule_path, ) +from napalm_device_types.models import FixStepOutcomeDict ALICE_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAlice alice@laptop" OWN_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQOwn own@desktop" @@ -639,3 +640,40 @@ def test_an_account_that_cannot_be_read_back_is_said(host: _Host, driver: _Drive assert result["account"] is None assert "[read back] cannot ask sudo about alice" in result["output"] + + +# -- a key store of the driver's own --------------------------------------------------- + + +class _OwnKeyStore(_Driver): + """A driver whose host keeps keys in a place of its own.""" + + def __init__(self, host: _Host) -> None: + super().__init__(host) + self.store: dict[str, list[str]] = {} + + def _account_read_keys(self, name: str, home: str) -> list[str] | None: + return list(self.store.get(name, [])) + + def _account_add_key(self, name: str, home: str, public_key: str) -> FixStepOutcomeDict: + self.store.setdefault(name, []).append(f"{home} {public_key}") + return {"ok": True, "output": "stored"} + + def _account_remove_keys(self, name: str, home: str) -> FixStepOutcomeDict: + self.store.pop(name, None) + return {"ok": True, "output": "dropped"} + + +def test_a_driver_can_keep_the_keys_elsewhere(host: _Host) -> None: + driver = _OwnKeyStore(host) + + added = driver.add_local_account("alice", ALICE_KEY) + assert added["success"], added["output"] + assert driver.store == {"alice": [f"/home/alice {ALICE_KEY}"]} + assert added["account"] is not None + assert added["account"]["authorized_keys"] == [f"/home/alice {ALICE_KEY}"] + assert not [p for p in host.files if p.startswith("/home/")] + + locked = driver.lock_local_account("alice") + assert locked["success"], locked["output"] + assert driver.store == {} From 2846824a21958337b56248c1bd09ea89e9145098 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sat, 31 Oct 2026 23:31:05 +0100 Subject: [PATCH 31/50] feat: a host may add a key file sshd reads (NetOrk/netork#1793) _account_key_files says which files hold an account's keys; the default is ~/.ssh/authorized_keys and authorized_keys2. OpenMediaVault adds the file it deploys under /var/lib/openmediavault/ssh/authorized_keys, so reading an account back sees every key sshd would take. --- CHANGELOG.md | 2 +- napalm_device_types/local_accounts.py | 18 +++++++++--------- tests/test_local_accounts.py | 13 +++++++++++++ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b134692..a3df652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ recorded here; their history is the git log. the key files are read and written as the account itself, and the sudo rule is a drop-in of its own, checked with `visudo` and `sudo -l`. A driver supplies the commands that create, open and lock an account and take it out of a group, and may - replace where the keys are kept (`_account_read_keys`, `_account_add_key`, + replace where the keys are kept (`_account_key_files`, `_account_read_keys`, `_account_add_key`, `_account_remove_keys`; default `~/.ssh/authorized_keys`) (NetOrk/netork#1793). - `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py index f4d23d4..d579bc7 100644 --- a/napalm_device_types/local_accounts.py +++ b/napalm_device_types/local_accounts.py @@ -197,8 +197,8 @@ class LocalAccountsMixin: # -- what a driver supplies ------------------------------------------------------ # # ``_account_*`` are the driver's: the five below it must supply, the key - # store (``_account_read_keys``, ``_account_add_key``, ``_account_remove_keys``) - # it may replace. ``_accounts_*`` are this mixin's own. + # store (``_account_key_files``, ``_account_read_keys``, ``_account_add_key``, + # ``_account_remove_keys``) it may replace. ``_accounts_*`` are this mixin's own. def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: """The commands that create *name* with a home and :attr:`ACCOUNT_SHELL`. @@ -276,14 +276,14 @@ class LocalAccountsMixin: raise RuntimeError(f"cannot ask sudo about {name}: {_said(result)}") return bool(_RUN_ALL_RE.search(result.stdout)) - def _account_read_keys(self, name: str, home: str) -> list[str] | None: - """The lines of *name*'s key files, read as *name*; None when it cannot. + def _account_key_files(self, name: str, home: str) -> tuple[str, ...]: + """Every file sshd reads *name*'s keys from; a host that adds one of its own says so.""" + return authorized_keys_paths(home) - Every key sshd would take for *name*: a driver whose host keeps keys in - a place of its own as well reads that too. - """ + def _account_read_keys(self, name: str, home: str) -> list[str] | None: + """The lines of :meth:`_account_key_files`, read as *name*; None when it cannot.""" lines: list[str] = [] - for path in authorized_keys_paths(home): + for path in self._account_key_files(name, home): there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) if there.exit_code != 0: continue @@ -555,7 +555,7 @@ class LocalAccountsMixin: return None def _account_remove_keys(self, name: str, home: str) -> FixStepOutcomeDict: - """Remove *name*'s key files, as *name*.""" + """Remove *name*'s key files in *home*, as *name*; then no key may be left.""" paths = authorized_keys_paths(home) failed = self._accounts_run_as(name, [["rm", "-f", "--", *paths]]) if failed is not None: diff --git a/tests/test_local_accounts.py b/tests/test_local_accounts.py index c9ed53c..30f1363 100644 --- a/tests/test_local_accounts.py +++ b/tests/test_local_accounts.py @@ -677,3 +677,16 @@ def test_a_driver_can_keep_the_keys_elsewhere(host: _Host) -> None: locked = driver.lock_local_account("alice") assert locked["success"], locked["output"] assert driver.store == {} + + +def test_a_key_file_the_host_adds_is_read_too(host: _Host) -> None: + class _ExtraFile(_Driver): + def _account_key_files(self, name: str, home: str) -> tuple[str, ...]: + return (*super()._account_key_files(name, home), f"/home/{name}/extra") + + _add_alice(host, keys=OWN_KEY + "\n") + host.files["/home/alice/extra"] = ALICE_KEY + "\n" + + [account] = _ExtraFile(host).get_local_accounts(["alice"]) + + assert account["authorized_keys"] == [OWN_KEY, ALICE_KEY] From 5fffb3b886f0909addbc634e797fb0f1131502c1 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 00:24:11 +0100 Subject: [PATCH 32/50] feat: the managing client and sshd's per-source penalties (NetOrk/netork#1800) OpenSSH 9.8+ penalises a source whose connections do not authenticate -- a scanner's probes -- and then drops every connection from it, the managing client's included. - OpenSshServerMixin.exempt_from_ssh_penalties puts the address the host sees the client at ($SSH_CONNECTION) into PerSourcePenaltyExemptList, next to the host's own exemptions, in its own drop-in. Only where sshd has the penalties (an older one refuses the directive) and includes the drop-in directory; checked with sshd -t, sshd -T and a new session, and taken back on any failure. The check-and-restore step of harden_ssh_server is shared for it. - retry_refused tries a connection again after 1, 3, 7 and 15 s while sshd closes it before its banner; HostKeyCheck.connect uses it. --- CHANGELOG.md | 12 ++ napalm_device_types/ssh_host_keys.py | 59 ++++++- napalm_device_types/ssh_server.py | 227 ++++++++++++++++++++++-- tests/test_ssh_penalties.py | 250 +++++++++++++++++++++++++++ tests/test_ssh_refused.py | 168 ++++++++++++++++++ 5 files changed, 698 insertions(+), 18 deletions(-) create mode 100644 tests/test_ssh_penalties.py create mode 100644 tests/test_ssh_refused.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b04273..7a297b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,18 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `OpenSshServerMixin.exempt_from_ssh_penalties`: puts the address the host sees + the managing client at (`$SSH_CONNECTION`) into `PerSourcePenaltyExemptList`, + in `/etc/ssh/sshd_config.d/00-netork-penalty-exempt.conf`, next to the host's + own exemptions. Only where sshd has per-source penalties (OpenSSH 9.8+; an older + one refuses the directive) and sshd_config includes the drop-in directory; + checked with `sshd -t` and `sshd -T`, then a new session, and taken back on any + failure. Helpers: `parse_penalty_exemptions`, `client_address`, `is_exempt` + (NetOrk/netork#1800). +- `retry_refused` and `refused_before_banner` (`napalm_device_types.ssh_host_keys`): + a connection sshd closes before its banner -- what OpenSSH 9.8+ does to a source + it has just penalised -- is tried again after 1, 3, 7 and 15 s. + `HostKeyCheck.connect` does so itself (NetOrk/netork#1800). - `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how old an entry is. `0.0` means "confirmed just now" and is no stand-in for it diff --git a/napalm_device_types/ssh_host_keys.py b/napalm_device_types/ssh_host_keys.py index a9d2df4..4413cf6 100644 --- a/napalm_device_types/ssh_host_keys.py +++ b/napalm_device_types/ssh_host_keys.py @@ -27,6 +27,11 @@ A caller that passes neither keeps working as before (first use, every time), and a driver that ignores both still connects for a caller that passes them: old and new versions on either side do not break each other. +A connection sshd turns away before its banner -- OpenSSH 9.8+ does that to a +source it has just penalised -- is tried again after a pause +(:func:`retry_refused`, :data:`REFUSED_RETRY_WAITS`); :meth:`HostKeyCheck.connect` +does so itself, a netmiko driver wraps its ``ConnectHandler`` in it. + :class:`HostKeyCheck` is the one implementation, for a paramiko ``SSHClient`` (:meth:`~HostKeyCheck.connect`) and for netmiko (:meth:`~HostKeyCheck.netmiko`, then :meth:`~HostKeyCheck.observe`). Like :mod:`napalm_device_types.channel` it @@ -40,7 +45,8 @@ import hashlib import logging import os import tempfile -from collections.abc import Iterator, Mapping +import time +from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from typing import Any @@ -54,6 +60,13 @@ KNOWN_KEYS_ARG = "ssh_host_keys" #: The ``optional_args`` key that carries the callable every seen key goes to. OBSERVER_ARG = "ssh_host_key_observer" +#: The waits before each new attempt when sshd turns a connection away before its +#: banner. OpenSSH 9.8+ does that to every connection from a source it has just +#: penalised (``PerSourcePenalties``) -- one whose connections did not +#: authenticate, a scanner's probes -- for seconds to minutes. 26 s in all +#: outlasts the 16 s penalty seen on a new Ubuntu 26.04 host (NetOrk/netork#1800). +REFUSED_RETRY_WAITS = (1.0, 3.0, 7.0, 15.0) + #: The netmiko arguments this module decides; a caller's own values give way. _NETMIKO_HOST_KEY_ARGS = ("ssh_strict", "system_host_keys", "alt_host_keys", "alt_key_file") @@ -185,6 +198,48 @@ def _refused_key(exc: BaseException) -> Any | None: return None +def refused_before_banner(exc: BaseException) -> bool: + """Whether *exc*, or what it wraps, is sshd closing the connection before its banner. + + paramiko says "Error reading SSH protocol banner" (netmiko and drivers wrap + it), with the connection's EOF or reset underneath. A failed login, a + silent host and a closed port are something else: waiting does not end them. + """ + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + if "protocol banner" in str(current): + return True + stack.extend([current.__cause__, current.__context__]) + return False + + +def retry_refused[T]( + attempt: Callable[[], T], + *, + waits: tuple[float, ...] = REFUSED_RETRY_WAITS, + sleep: Callable[[float], None] | None = None, +) -> T: + """*attempt*, again after each of *waits* while sshd turns it away before its banner. + + Any other error, and the last refusal, go to the caller unchanged. + """ + pause = sleep or time.sleep + for wait in waits: + try: + return attempt() + except Exception as exc: + if not refused_before_banner(exc): + raise + logger.info("SSH connection turned away before its banner; again in %s s", wait) + pause(wait) + return attempt() + + class HostKeyCheck: """How one driver connection verifies its host's key. @@ -220,7 +275,7 @@ class HostKeyCheck: """``client.connect()`` to this host with *kwargs*, verified and reported.""" self.prepare(client) with self.refusing(): - client.connect(hostname=self.host, port=self.port, **kwargs) + retry_refused(lambda: client.connect(hostname=self.host, port=self.port, **kwargs)) return self.observe(client.get_transport()) # ------------------------------------------------------------------ diff --git a/napalm_device_types/ssh_server.py b/napalm_device_types/ssh_server.py index a20c9db..8a67d55 100644 --- a/napalm_device_types/ssh_server.py +++ b/napalm_device_types/ssh_server.py @@ -30,10 +30,19 @@ Only without such an ``Include`` is sshd_config itself rewritten, after a copy. 3. After the reload, *verify* -- the caller's -- opens a new session. The running one survives a reload, so if the new one cannot get in, this one puts the old state back and reloads again. + +**The managing client is exempt from per-source penalties** where sshd has them +(OpenSSH 9.8+, :meth:`OpenSshServerMixin.exempt_from_ssh_penalties`). sshd +penalises a source whose connections do not authenticate -- a scanner's probes +-- and then drops every connection from it, the managing session's included. +The address is the one the host sees (``$SSH_CONNECTION``); it joins the +host's own ``PerSourcePenaltyExemptList`` in :data:`PENALTY_DROPIN_PATH`, with +the same three checks. An older sshd is not touched: it refuses the directive. """ from __future__ import annotations +import ipaddress import time from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field @@ -57,6 +66,9 @@ MAIN_CONFIG = "/etc/ssh/sshd_config" DROPIN_DIR = "/etc/ssh/sshd_config.d" #: ``00-`` so it sorts before every other drop-in: sshd keeps the first value it reads. DROPIN_PATH = f"{DROPIN_DIR}/00-netork-ssh-hardening.conf" +#: Where :meth:`OpenSshServerMixin.exempt_from_ssh_penalties` names the managing +#: client; ``00-`` like the hardening drop-in, since sshd keeps the first value. +PENALTY_DROPIN_PATH = f"{DROPIN_DIR}/00-netork-penalty-exempt.conf" #: What marks a line this module disabled in sshd_config. DISABLED_PREFIX = "# netork: " @@ -65,6 +77,11 @@ _HEADER = ( "# taken out of what this server offers. Delete this file to undo it.\n" ) +_PENALTY_HEADER = ( + "# Written by netOrk: the address netOrk manages this host from, exempt from\n" + "# sshd's per-source penalties (PerSourcePenalties). Delete this file to undo it.\n" +) + #: Host key type, as ``ssh-keygen -l`` prints it -> the algorithms that key serves. #: Certificate algorithms are left out: they also need a HostCertificate. _KEY_TYPE_ALGORITHMS: dict[str, tuple[str, ...]] = { @@ -302,6 +319,56 @@ def render_main_config(text: str, keep: Mapping[str, Sequence[str]], kinds: Iter return render_dropin(keep, wanted) + "\n".join(out) + ("\n" if body.endswith("\n") else "") +def parse_penalty_exemptions(text: str) -> list[str] | None: + """The ``persourcepenaltyexemptlist`` from ``sshd -T`` output; None without penalties. + + Per-source penalties came with OpenSSH 9.8. An older sshd prints neither + line and refuses ``PerSourcePenaltyExemptList`` outright -- it does not + start with it in its configuration -- so None means: write nothing. + """ + penalties = False + exempt: list[str] = [] + for raw in text.splitlines(): + parts = raw.strip().split(None, 1) + if not parts: + continue + word = parts[0].lower() + if word == "persourcepenalties": + penalties = True + elif word == "persourcepenaltyexemptlist" and len(parts) == 2: + value = parts[1].strip() + exempt = [] if value.lower() == "none" else [a for a in value.split(",") if a] + return exempt if penalties else None + + +def client_address(ssh_connection: str) -> str | None: + """The client's address in ``$SSH_CONNECTION`` (``client port server port``), or None.""" + parts = ssh_connection.split() + if len(parts) != 4: + return None + try: + return str(ipaddress.ip_address(parts[0])) + except ValueError: + return None + + +def is_exempt(address: str, exempt: Iterable[str]) -> bool: + """Whether *address* is one of *exempt*'s addresses or inside one of its networks.""" + ip = ipaddress.ip_address(address) + for entry in exempt: + try: + if ip in ipaddress.ip_network(entry, strict=False): + return True + except ValueError: + continue + return False + + +def render_penalty_dropin(exempt: Sequence[str]) -> str: + """The drop-in stating the full exempt list.""" + return _PENALTY_HEADER + f"PerSourcePenaltyExemptList {','.join(exempt)}\n" + + @dataclass class SshHardeningResult: """What :meth:`OpenSshServerMixin.harden_ssh_server` did. @@ -319,6 +386,30 @@ class SshHardeningResult: message: str = "" +@dataclass +class SshPenaltyExemption: + """What :meth:`OpenSshServerMixin.exempt_from_ssh_penalties` did. + + *address* is the one the host sees the managing client at. *applied* is + True only when sshd uses the exemption and a new session got in; + *rolled_back* says the host was changed and then put back. + """ + + address: str | None = None + applied: bool = False + rolled_back: bool = False + path: str | None = None + message: str = "" + + +def _unchanged( + result: SshPenaltyExemption, message: str, say: Callable[[str], None] +) -> SshPenaltyExemption: + result.message = message + say(message) + return result + + class OpenSshServerMixin: """Adds :meth:`harden_ssh_server` to a driver that can run commands as root on an OpenSSH host. @@ -484,7 +575,25 @@ class OpenSshServerMixin: if main is None: raise RuntimeError(f"cannot read {MAIN_CONFIG}") restore = self._write_change(plan, main, result, say) - return self._apply_or_restore(plan, kinds, result, restore, verify, say) + + def ignored() -> str | None: + now = self.get_ssh_server_algorithms() + names = [SSH_ALGORITHM_CLASSES[k][1] for k in kinds if now[k] != plan.keep[k]] + if not names: + return None + return ( + f"sshd does not use the new {', '.join(names)}: a setting read before" + f" {result.path} decides them" + ) + + return self._apply_or_restore( + result, + restore, + verify, + say, + ignored=ignored, + done="The server no longer offers the selected algorithms.", + ) def _write_change( self, @@ -520,16 +629,23 @@ class OpenSshServerMixin: return restore_main - def _apply_or_restore( + def _apply_or_restore[R: (SshHardeningResult, SshPenaltyExemption)]( self, - plan: SshHardeningPlan, - kinds: list[str], - result: SshHardeningResult, + result: R, restore: Callable[[], None], verify: Callable[[], bool] | None, say: Callable[[str], None], - ) -> SshHardeningResult: - def back(message: str, reload: bool) -> SshHardeningResult: + *, + ignored: Callable[[], str | None], + done: str, + ) -> R: + """Check what was written, reload, verify; put the old state back on any failure. + + *ignored* reads ``sshd -T`` after ``sshd -t`` and says why sshd does not + use the change, or None when it does; *done* is the message of success. + """ + + def back(message: str, reload: bool) -> R: say(message) say("Putting the previous configuration back") try: @@ -549,14 +665,9 @@ class OpenSshServerMixin: complaint = self._sshd_accepts() if complaint: return back(f"sshd rejected the new configuration: {complaint}", reload=False) - now = self.get_ssh_server_algorithms() - ignored = [SSH_ALGORITHM_CLASSES[k][1] for k in kinds if now[k] != plan.keep[k]] - if ignored: - return back( - f"sshd does not use the new {', '.join(ignored)}: a setting read before" - f" {result.path} decides them", - reload=False, - ) + why = ignored() + if why: + return back(why, reload=False) say("Reloading sshd") self._reload_ssh_server() if verify is not None: @@ -566,6 +677,90 @@ class OpenSshServerMixin: except Exception as exc: # noqa: BLE001 - whatever went wrong, the host goes back return back(f"The change failed: {exc}", reload=True) result.applied = True - result.message = "The server no longer offers the selected algorithms." + result.message = done say(result.message) return result + + # ── the penalty exemption ─────────────────────────────────────────── + + def exempt_from_ssh_penalties( + self, + *, + verify: Callable[[], bool] | None = None, + log: Callable[[str], None] | None = None, + ) -> SshPenaltyExemption: + """Exempt the address this session comes from from sshd's per-source penalties. + + OpenSSH 9.8+ penalises a source whose connections do not authenticate -- + a scanner's probes -- and then drops every connection from it, the + managing client's included (NetOrk/netork#1800). The address is the + one the host sees (``$SSH_CONNECTION``), so NAT and relays do not + matter. The host's own exemptions stay: the new list is the one sshd + uses now plus this address. + + Only into the drop-in directory: a sshd_config that does not include it + is left alone, and the result says so. An sshd before 9.8 is not + touched -- it would not start with the directive. Checked like + :meth:`harden_ssh_server`: ``sshd -t``, ``sshd -T`` shows the address, + reload, then *verify*; on any failure the previous state goes back. + + :raises RuntimeError: if the session names no client address, or + ``sshd -T`` or sshd_config cannot be read. + """ + say = log or (lambda _line: None) + result = SshPenaltyExemption() + session = self.run_command('printf %s "$SSH_CONNECTION"', privileged=False, timeout=30) + address = client_address(session.stdout) + if address is None: + raise RuntimeError("the session names no client address ($SSH_CONNECTION)") + result.address = address + say(f"netOrk connects from {address}") + say("Reading sshd's per-source penalties (sshd -T)") + exempt = parse_penalty_exemptions(self._sshd_effective_text()) + if exempt is None: + return _unchanged( + result, + "Nothing to change: this sshd has no per-source penalties (before OpenSSH 9.8).", + say, + ) + if is_exempt(address, exempt): + return _unchanged(result, f"Nothing to change: {address} is already exempt.", say) + main = self._ssh_read(MAIN_CONFIG) + if main is None: + raise RuntimeError(f"cannot read {MAIN_CONFIG}") + if not has_dropin_include(main): + return _unchanged( + result, + f"Not changed: {MAIN_CONFIG} does not include {DROPIN_DIR}," + " where the exemption would go.", + say, + ) + + previous = self._ssh_read(PENALTY_DROPIN_PATH) + self._ssh_run(f"mkdir -p -- {quote(DROPIN_DIR)}") + self._ssh_write(PENALTY_DROPIN_PATH, render_penalty_dropin([*exempt, address])) + result.path = PENALTY_DROPIN_PATH + say(f"Wrote {PENALTY_DROPIN_PATH}") + + def restore() -> None: + if previous is None: + self._ssh_remove(PENALTY_DROPIN_PATH) + else: + self._ssh_write(PENALTY_DROPIN_PATH, previous) + + def ignored() -> str | None: + if is_exempt(address, parse_penalty_exemptions(self._sshd_effective_text()) or []): + return None + return ( + "sshd does not use the new PerSourcePenaltyExemptList: a setting read before" + f" {PENALTY_DROPIN_PATH} decides it" + ) + + return self._apply_or_restore( + result, + restore, + verify, + say, + ignored=ignored, + done=f"{address} is exempt from sshd's per-source penalties.", + ) diff --git a/tests/test_ssh_penalties.py b/tests/test_ssh_penalties.py new file mode 100644 index 0000000..69d16dc --- /dev/null +++ b/tests/test_ssh_penalties.py @@ -0,0 +1,250 @@ +"""Exempting the managing client from sshd's per-source penalties (NetOrk/netork#1800). + +OpenSSH 9.8 penalises a source whose connections do not authenticate and then +drops every connection from it. A security scanner's probes earn the managing +client exactly that, and its next session -- a poll, an Ansible run -- is +turned away. ``exempt_from_ssh_penalties`` puts the address the host sees the +client at into ``PerSourcePenaltyExemptList``, where that directive exists. + +The real outputs this is held to were taken from Debian 13 (OpenSSH 10.0p2) and +Debian 12 (OpenSSH 9.2p1, which knows no penalties and rejects the directive). +""" + +from __future__ import annotations + +import re + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.ssh_server import ( + DROPIN_DIR, + MAIN_CONFIG, + PENALTY_DROPIN_PATH, + OpenSshServerMixin, + client_address, + is_exempt, + parse_penalty_exemptions, +) + +#: ``sshd -T | grep -i penalt`` on Debian 13, OpenSSH 10.0p2, as installed. +DEBIAN_13 = ( + "persourcepenaltyexemptlist none\n" + "persourcepenalties crash:90 authfail:5 noauth:1 grace-exceeded:10 refuseconnection:10" + " max:600 min:15 max-sources4:65536 max-sources6:65536 overflow:permissive" + " overflow6:permissive\n" +) + + +# ── reading ────────────────────────────────────────────────────────────── + + +def test_a_new_sshd_has_penalties_and_no_exemption(): + assert parse_penalty_exemptions(DEBIAN_13) == [] + + +def test_the_exemptions_sshd_uses_are_read(): + text = DEBIAN_13.replace("none", "10.40.100.10,192.0.2.0/24") + + assert parse_penalty_exemptions(text) == ["10.40.100.10", "192.0.2.0/24"] + + +def test_an_sshd_before_9_8_has_no_penalties(): + """It prints neither line -- and refuses the directive, so nothing may be written.""" + assert parse_penalty_exemptions("port 22\nmacs hmac-sha2-256\n") is None + + +@pytest.mark.parametrize( + ("value", "address"), + [ + ("10.40.100.10 41866 10.40.100.118 22", "10.40.100.10"), + ("fd00::10 41866 fd00::118 22", "fd00::10"), + ("", None), + ("not-an-address 1 2 3", None), + ("10.40.100.10 41866", None), + ], +) +def test_the_client_address_comes_from_ssh_connection(value, address): + assert client_address(value) == address + + +def test_an_address_inside_an_exempt_network_is_exempt(): + assert is_exempt("10.40.100.10", ["192.0.2.0/24", "10.40.100.0/24"]) + assert not is_exempt("10.40.100.10", ["10.40.100.11", "bogus"]) + + +# ── the mixin, against a fake host ─────────────────────────────────────── + + +class PenaltyHost(OpenSshServerMixin): + """An sshd that reads the main config and the drop-in directory, first value wins.""" + + def __init__( + self, + *, + main: str = f"Include {DROPIN_DIR}/*.conf\nPort 22\n", + penalties: bool = True, + client: str = "10.40.100.10 41866 10.40.100.118 22", + ) -> None: + self.files: dict[str, str] = {MAIN_CONFIG: main} + self.penalties = penalties + self.client = client + self.syntax_ok = True + self.reloads = 0 + self.ran: list[str] = [] + + def _config_lines(self) -> list[str]: + lines: list[str] = [] + for line in self.files[MAIN_CONFIG].splitlines(): + if re.match(r"\s*include\s", line, re.I): + for path in sorted(p for p in self.files if p.startswith(DROPIN_DIR + "/")): + lines += self.files[path].splitlines() + elif re.match(r"\s*match\s", line, re.I): + break + else: + lines.append(line) + return lines + + def exempt(self) -> str: + for line in self._config_lines(): + parts = line.split() + if len(parts) == 2 and parts[0].lower() == "persourcepenaltyexemptlist": + return parts[1] + return "none" + + def run_command(self, command, *, privileged=False, timeout=60, stdin=None): + self.ran.append(command) + if "SSH_CONNECTION" in command: + assert not privileged, "it is the login session's, not sudo's" + return CommandResult(self.client, "", 0) + if "mktemp" in command: + path = f"/tmp/netork-ssh.{len(self.files)}" + self.files[path] = (stdin or b"").decode() + return CommandResult(path, "", 0) + assert privileged, command + m = re.search(r"install -m \"\$m\" -- '?([^' ]+)'? '?([^' ]+)'?", command) + if m: + target = re.search(r"mv -f -- '?[^' ]+'? '?([^' ;]+)'?", command).group(1) + self.files[target] = self.files.pop(m.group(1)) + return CommandResult("", "", 0) + if "sshd -T" in command: + if not self.penalties: + return CommandResult("port 22\n", "", 0) + return CommandResult(DEBIAN_13.replace("none", self.exempt()), "", 0) + if "sshd -t" in command: + return CommandResult( + "", "" if self.syntax_ok else "bad line 1", 0 if self.syntax_ok else 255 + ) + m = re.match(r"cat -- '?([^' ]+)'?$", command) + if m: + path = m.group(1) + if path in self.files: + return CommandResult(self.files[path], "", 0) + return CommandResult("", "No such file", 1) + m = re.match(r"rm -f -- '?([^' ]+)'?$", command) + if m: + self.files.pop(m.group(1), None) + return CommandResult("", "", 0) + if "reload" in command or "HUP" in command: + self.reloads += 1 + return CommandResult("", "", 0) + if command.startswith("mkdir"): + return CommandResult("", "", 0) + raise AssertionError(f"unexpected command: {command}") + + +def test_the_client_address_is_exempted_in_a_dropin(): + host = PenaltyHost() + lines: list[str] = [] + + result = host.exempt_from_ssh_penalties(log=lines.append) + + assert result.applied and not result.rolled_back + assert result.address == "10.40.100.10" + assert result.path == PENALTY_DROPIN_PATH + assert host.exempt() == "10.40.100.10" + assert host.reloads == 1 + assert lines + + +def test_the_hosts_own_exemptions_stay(): + host = PenaltyHost( + main=f"Include {DROPIN_DIR}/*.conf\nPerSourcePenaltyExemptList 192.0.2.0/24\n" + ) + + result = host.exempt_from_ssh_penalties() + + assert result.applied + assert host.exempt() == "192.0.2.0/24,10.40.100.10" + + +def test_nothing_is_written_to_an_sshd_without_penalties(): + """OpenSSH before 9.8 does not start with the directive in its config.""" + host = PenaltyHost(penalties=False) + + result = host.exempt_from_ssh_penalties() + + assert not result.applied and not result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + assert host.reloads == 0 + assert "9.8" in result.message + + +def test_nothing_changes_when_the_address_is_already_exempt(): + host = PenaltyHost( + main=f"Include {DROPIN_DIR}/*.conf\nPerSourcePenaltyExemptList 10.40.100.0/24\n" + ) + + result = host.exempt_from_ssh_penalties() + + assert not result.applied + assert PENALTY_DROPIN_PATH not in host.files + assert "already" in result.message + + +def test_an_sshd_config_without_the_include_is_left_alone(): + host = PenaltyHost(main="Port 22\n") + + result = host.exempt_from_ssh_penalties() + + assert not result.applied + assert host.files[MAIN_CONFIG] == "Port 22\n" + assert DROPIN_DIR in result.message + + +def test_a_rejected_config_is_taken_back(): + host = PenaltyHost() + host.syntax_ok = False + + result = host.exempt_from_ssh_penalties() + + assert result.rolled_back and not result.applied + assert PENALTY_DROPIN_PATH not in host.files + + +def test_an_exemption_sshd_does_not_use_is_taken_back(): + """A drop-in read before ours sets the directive; ours is valid and ignored.""" + host = PenaltyHost() + host.files[f"{DROPIN_DIR}/00-aaa.conf"] = "PerSourcePenaltyExemptList 192.0.2.1\n" + + result = host.exempt_from_ssh_penalties() + + assert result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + + +def test_it_is_taken_back_when_a_new_session_cannot_get_in(): + host = PenaltyHost() + + result = host.exempt_from_ssh_penalties(verify=lambda: False) + + assert result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + assert host.reloads == 2 + + +def test_a_session_without_a_client_address_raises(): + host = PenaltyHost(client="") + + with pytest.raises(RuntimeError): + host.exempt_from_ssh_penalties() diff --git a/tests/test_ssh_refused.py b/tests/test_ssh_refused.py new file mode 100644 index 0000000..f736006 --- /dev/null +++ b/tests/test_ssh_refused.py @@ -0,0 +1,168 @@ +"""A connection sshd turns away before the SSH banner is tried again (NetOrk/netork#1800). + +OpenSSH 9.8+ drops every connection from a source it has just penalised +(``PerSourcePenalties``): it accepts the TCP connection and closes it at once, +and paramiko reports "Error reading SSH protocol banner". The penalty lasts +seconds to minutes, so a connection that waits a little gets in. +""" + +from __future__ import annotations + +import socket +import threading + +import paramiko +import pytest +from napalm.base.exceptions import ConnectionException + +from napalm_device_types import ssh_host_keys +from napalm_device_types.ssh_host_keys import ( + REFUSED_RETRY_WAITS, + HostKeyCheck, + refused_before_banner, + retry_refused, +) + + +def _banner_error() -> paramiko.SSHException: + try: + try: + raise EOFError() + except EOFError as eof: + raise paramiko.SSHException("Error reading SSH protocol banner") from eof + except paramiko.SSHException as exc: + return exc + + +class TestWhatCountsAsRefused: + def test_paramikos_banner_error(self): + assert refused_before_banner(_banner_error()) + + def test_wrapped_by_a_driver(self): + try: + try: + raise _banner_error() + except paramiko.SSHException as exc: + raise ConnectionException(f"cannot connect: {exc}") from exc + except ConnectionException as wrapped: + assert refused_before_banner(wrapped) + + @pytest.mark.parametrize( + "exc", + [ + paramiko.AuthenticationException("Authentication failed."), + TimeoutError("timed out"), + ConnectionRefusedError(111, "Connection refused"), + ValueError("anything else"), + ], + ) + def test_not_a_failed_login_a_silent_host_or_a_closed_port(self, exc): + """Those do not end by waiting: retrying them only delays the error.""" + assert not refused_before_banner(exc) + + +class TestRetry: + def test_it_waits_and_tries_again(self): + slept: list[float] = [] + attempts = iter([_banner_error(), _banner_error(), "connected"]) + + def attempt(): + value = next(attempts) + if isinstance(value, BaseException): + raise value + return value + + assert retry_refused(attempt, sleep=slept.append) == "connected" + assert slept == list(REFUSED_RETRY_WAITS[:2]) + + def test_it_gives_up_with_the_last_error(self): + slept: list[float] = [] + + def attempt(): + raise _banner_error() + + with pytest.raises(paramiko.SSHException): + retry_refused(attempt, sleep=slept.append) + assert slept == list(REFUSED_RETRY_WAITS) + + def test_other_errors_are_not_retried(self): + slept: list[float] = [] + + def attempt(): + raise paramiko.AuthenticationException("Authentication failed.") + + with pytest.raises(paramiko.AuthenticationException): + retry_refused(attempt, sleep=slept.append) + assert slept == [] + + def test_the_waits_outlast_the_penalty_seen(self): + """16 s on a new Ubuntu 26.04 VM on .50.""" + assert sum(REFUSED_RETRY_WAITS) > 16 + + +class _PenalisingServer: + """A TCP server that closes the first *drops* connections at once, as a penalised + source's are, and then speaks SSH with any password.""" + + def __init__(self, drops: int) -> None: + self.drops = drops + self.listener = socket.create_server(("127.0.0.1", 0)) + self.port = self.listener.getsockname()[1] + self.key = paramiko.ECDSAKey.generate() + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + while True: + try: + conn, _ = self.listener.accept() + except OSError: + return + if self.drops: + self.drops -= 1 + conn.close() + continue + transport = paramiko.Transport(conn) + transport.add_server_key(self.key) + + class _Any(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + try: + transport.start_server(server=_Any()) + except Exception: + pass + self.transport = transport + return + + def close(self) -> None: + self.listener.close() + transport = getattr(self, "transport", None) + if transport: + transport.close() + + +def test_host_key_check_connects_once_the_penalty_is_over(monkeypatch): + server = _PenalisingServer(drops=2) + slept: list[float] = [] + monkeypatch.setattr(ssh_host_keys.time, "sleep", slept.append) + client = paramiko.SSHClient() + try: + HostKeyCheck({}, "127.0.0.1", server.port).connect( + client, + username="admin", + password="s3cret", + look_for_keys=False, + allow_agent=False, + timeout=5, + banner_timeout=5, + ) + assert client.get_transport().is_authenticated() + assert slept == list(REFUSED_RETRY_WAITS[:2]) + finally: + client.close() + server.close() From 3b6c5f958550c6f3593fb00c2cd0f510fb0c0fe7 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 07:58:05 +0100 Subject: [PATCH 33/50] fix(apt_proxy): https sources take the http proxy too (NetOrk/netork#1811) apt's https method asks Acquire::https::Proxy and then Acquire::http::Proxy, for the host and then for all. An 00proxy with the http line alone therefore sent download.docker.com and every other https source to apt-cacher-ng, which answers 403 CONNECT denied without a PassThroughPattern, and the whole upgrade failed. - The report carries https_proxy, https_per_host and https_settings; https_proxy_for(report, host) is the proxy apt takes for one https host. - check_https_source() fetches one https source the way apt does. - configure_package_proxy gains an https step: one https source through the new proxy. Where the proxy refuses it, 00proxy also says Acquire::https::Proxy "DIRECT", later https lines to the same proxy are commented out, and the source is fetched direct. An HTTP error from the mirror counts as let through. - apt_fetch_command takes https URLs; comment_out_command takes a scheme. --- CHANGELOG.md | 17 ++ napalm_device_types/__init__.py | 2 + napalm_device_types/apt_proxy.py | 272 ++++++++++++++++++++++---- tests/test_apt_proxy.py | 319 +++++++++++++++++++++++++++++-- 4 files changed, 555 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8279e69..3312e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,23 @@ recorded here; their history is the git log. link, owned by the login user. It reports every file's lines before and after, how many were removed and which files are root's again. Nothing goes to stdin. +### Fixed +- `AptProxyMixin` knows that apt's https sources take the http proxy too: apt's https + method asks `Acquire::https::Proxy` and then `Acquire::http::Proxy`, for the host and + then for all. An `00proxy` with the http line alone sent download.docker.com and + every other https source to apt-cacher-ng, which answers `403 CONNECT denied` without + a `PassThroughPattern`, and the whole upgrade failed (NetOrk/netork#1811). + - The report carries `https_proxy`, `https_per_host` and `https_settings`; + `https_proxy_for(report, host)` is the proxy apt takes for one https host. + - `check_https_source()` fetches one https source's `InRelease` the way apt does; + None where no https source goes through a proxy. + - `configure_package_proxy` gains an `https` step: one https source through the new + proxy. Where the proxy refuses it, `00proxy` also says `Acquire::https::Proxy + "DIRECT";`, later https lines to the same proxy are commented out, and the source + is fetched direct. An HTTP error from the mirror counts as let through. + - `apt_fetch_command` takes https URLs; `direct` sets the host's https setting too. + - `comment_out_command(..., scheme="https")`. + ## [4.22.0] - 2026-10-23 ### Added diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 35e95ed..a3d1e8f 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -72,6 +72,7 @@ from napalm_device_types.access_point import AccessPointDriver from napalm_device_types.apt_proxy import ( APT_PROXY_COMMAND, AptProxyMixin, + https_proxy_for, parse_apt_fetch, parse_apt_proxy, same_proxy_url, @@ -252,6 +253,7 @@ __all__ = [ "PackageProxyMixin", "APT_PROXY_COMMAND", "AptProxyMixin", + "https_proxy_for", "parse_apt_fetch", "parse_apt_proxy", "same_proxy_url", diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index 7cc7b6f..e8e933a 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -12,12 +12,18 @@ Debian installer writes its proxy into ``apt.conf``, Ubuntu's into (NetOrk/netork#1397). ``DIRECT`` for the proxy, or for one host (``Acquire::http::Proxy::``), means no proxy there. +**https** goes through the http proxy too. apt's https method asks +``Acquire::https::Proxy`` first and ``Acquire::http::Proxy`` after it, for one +host and then for all (NetOrk/netork#1811). An http proxy alone therefore takes +every https source as well, and a cache that does not tunnel them answers 403. + **Where it is set** is every line of apt's files that sets the http proxy, in the -order apt reads them. A line of the form ``Acquire::http::Proxy "…";`` can be -commented out; a setting inside a block (``Acquire::http { Proxy "…"; };``) is -named but left alone. apt skips a file in ``apt.conf.d`` whose name has other -characters than letters, digits, ``_``, ``-`` and ``.``, or an extension other -than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); so is it here. +order apt reads them; the https proxy's lines are named apart. A line of the form +``Acquire::http::Proxy "…";`` can be commented out; a setting inside a block +(``Acquire::http { Proxy "…"; };``) is named but left alone. apt skips a file in +``apt.conf.d`` whose name has other characters than letters, digits, ``_``, ``-`` +and ``.``, or an extension other than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); +so is it here. **Whether a proxy answers** is a fetch with apt's own downloader, ``apt-helper download-file``, which is on every apt host and needs no root: @@ -29,13 +35,15 @@ page is the right one, the caller says (``expect``). out every other line that sets a different one, reads the result back from ``apt-config``, and fetches the ``InRelease`` of one http source through it. The last step is what proves it: apt reached a mirror through the proxy it is now set -to. The source chosen is one apt does not fetch ``DIRECT``. +to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one +https source the same way; where the proxy refuses it, https goes ``DIRECT``. """ from __future__ import annotations import re import shlex +from collections.abc import Callable from typing import TYPE_CHECKING from urllib.parse import urlsplit @@ -47,7 +55,7 @@ from napalm_device_types.terminal import strip_terminal_codes class AptProxySettingDict(TypedDict): - """One line of apt's configuration that sets the http proxy (``AptProxyMixin``).""" + """One line of apt's configuration that sets a proxy (``AptProxyMixin``).""" file: str line: int # 1-based @@ -68,6 +76,14 @@ class AptProxyReportDict(TypedDict): auto_detect: str | None #: Every line that sets it, in the order apt reads them; the last one wins. settings: list[AptProxySettingDict] + #: The proxy apt takes for https: ``Acquire::https::Proxy``, else the http one; + #: None for none or ``DIRECT``. + https_proxy: str | None + #: The per-host proxies https takes: ``Acquire::https::Proxy::``, else + #: ``Acquire::http::Proxy::``. + https_per_host: dict[str, str] + #: Every line that sets ``Acquire::https::Proxy``, in the order apt reads them. + https_settings: list[AptProxySettingDict] #: The base URIs of the host's sources over http and over https. http_sources: list[str] https_sources: list[str] @@ -85,6 +101,15 @@ class ProxyFetchDict(TypedDict): error: str +class HttpsSourceFetchDict(ProxyFetchDict): + """One https source fetched the way apt fetches it (``check_https_source``).""" + + #: What was fetched: the source's ``InRelease``. + url: str + #: The proxy apt took for it. + proxy: str + + _BEGIN = "APROXY_BEGIN" _END = "APROXY_END" _APT_HELPER = "/usr/lib/apt/apt-helper" @@ -98,7 +123,7 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") #: them early. APT_PROXY_COMMAND = ( "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " - "echo '[dump]'; apt-config dump 2>/dev/null | grep -i '^Acquire::http::Proxy'; " + "echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; " 'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; ' "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " '[ -f "$f" ] && grep -Hin proxy "$f"; done; ' @@ -108,12 +133,15 @@ APT_PROXY_COMMAND = ( "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" ) -_DUMP = re.compile(r'^Acquire::http::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) +_DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) _GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") _FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) _NOT_HTTP_PROXY = re.compile( r"^\s*(//|#)|Acquire::(https|ftp|socks)\b|::Proxy::|Proxy-Auto-Detect", re.I ) +_FLAT_HTTPS = re.compile(r'^\s*Acquire::https::Proxy\s+"([^"]*)"\s*;', re.I) +#: A line that sets the https proxy inside a block, all on the line itself. +_HTTPS_BLOCK = re.compile(r'^\s*Acquire::https\s*\{[^}]*\bProxy\s+"([^"]*)"', re.I) _IN_BLOCK = re.compile(r'\bProxy\s+"([^"]*)"', re.I) #: The names apt reads in ``apt.conf.d``: no extension, or ``.conf``. _PART_NAME = re.compile(r"^(?:[A-Za-z0-9_\-]+|[A-Za-z0-9_.\-]+\.conf)$") @@ -138,13 +166,22 @@ def _sections(output: str) -> dict[str, list[str]]: return sections -def _setting(line: str, main: bool) -> AptProxySettingDict | None: +def _grep_line(line: str, main: bool) -> tuple[str, int, str] | None: + """File, line number and text of one line grep printed; None for a file apt skips.""" found = _GREP.match(line) if not found: return None path, number, text = found.groups() if not main and not _PART_NAME.match(path.rsplit("/", 1)[-1]): return None + return path, int(number), text + + +def _setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found flat = _FLAT.match(text) if flat: return {"file": path, "line": int(number), "value": flat.group(1), "editable": True} @@ -156,22 +193,72 @@ def _setting(line: str, main: bool) -> AptProxySettingDict | None: return {"file": path, "line": int(number), "value": nested.group(1), "editable": False} -def _dump(lines: list[str]) -> tuple[str | None, dict[str, str], str | None]: - """The proxy, the per-host proxies and the auto-detect program in ``apt-config dump``.""" - proxy: str | None = None - per_host: dict[str, str] = {} - auto_detect: str | None = None +def _https_setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found + flat = _FLAT_HTTPS.match(text) + if flat: + return {"file": path, "line": number, "value": flat.group(1), "editable": True} + block = _HTTPS_BLOCK.match(text) + if block: + return {"file": path, "line": number, "value": block.group(1), "editable": False} + return None + + +class _Dump(TypedDict): + proxy: str | None + per_host: dict[str, str] + auto_detect: str | None + #: ``Acquire::https::Proxy`` as written; None when unset or empty. + https: str | None + https_per_host: dict[str, str] + + +def _dump(lines: list[str]) -> _Dump: + """The proxies, the per-host proxies and the auto-detect program in ``apt-config dump``.""" + dump: _Dump = { + "proxy": None, + "per_host": {}, + "auto_detect": None, + "https": None, + "https_per_host": {}, + } for found in (_DUMP.match(line.strip()) for line in lines): if not found: continue - suffix, value = found.groups() - if not suffix: - proxy = None if value.lower() in _NO_PROXY else value + scheme, suffix, value = found.groups() + https = scheme.lower() == "https" + if not suffix and https: + dump["https"] = value or None + elif not suffix: + dump["proxy"] = None if value.lower() in _NO_PROXY else value elif suffix.lower() == "-auto-detect": - auto_detect = value or None - else: - per_host[suffix[2:]] = value - return proxy, per_host, auto_detect + if not https: + dump["auto_detect"] = value or None + elif value: + dump["https_per_host" if https else "per_host"][suffix[2:]] = value + return dump + + +def _https_proxy(dump: _Dump) -> str | None: + """apt's https method takes its own setting and falls back to the http one.""" + if dump["https"] is None: + return dump["proxy"] + return None if dump["https"].lower() in _NO_PROXY else dump["https"] + + +def https_proxy_for(report: AptProxyReportDict, host: str) -> str | None: + """The proxy apt takes for an https source on *host*; None for ``DIRECT`` or none. + + The host's own setting first, https before http, then the one for all hosts. + """ + for per_host in (report["https_per_host"], report["per_host"]): + value = next((v for h, v in per_host.items() if h.lower() == host.lower()), None) + if value: + return None if value.lower() in _NO_PROXY else value + return report["https_proxy"] def parse_apt_proxy(output: str) -> AptProxyReportDict: @@ -180,19 +267,21 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: :raises ValueError: if the output carried no intact report. """ sections = _sections(output) - proxy, per_host, auto_detect = _dump(sections.get("dump", [])) - settings = [ - s + dump = _dump(sections.get("dump", [])) + lines = [ + (main, line) for main, key in ((False, "parts"), (True, "main")) - for s in (_setting(line, main) for line in sections.get(key, [])) - if s + for line in sections.get(key, []) ] sources = [line.strip() for line in sections.get("sources", [])] return { - "proxy": proxy, - "per_host": per_host, - "auto_detect": auto_detect, - "settings": settings, + "proxy": dump["proxy"], + "per_host": dump["per_host"], + "auto_detect": dump["auto_detect"], + "settings": [s for s in (_setting(line, main) for main, line in lines) if s], + "https_proxy": _https_proxy(dump), + "https_per_host": {**dump["per_host"], **dump["https_per_host"]}, + "https_settings": [s for s in (_https_setting(line, main) for main, line in lines) if s], "http_sources": [s for s in sources if s.startswith("http://")], "https_sources": [s for s in sources if s.startswith("https://")], } @@ -225,14 +314,17 @@ def apt_fetch_command(url: str, *, direct: bool = False, expect: str = "") -> st *direct* goes past any proxy apt is set to, to *url*'s host itself; without it the fetch takes apt's own way. *expect* is text the page should contain. - :raises ValueError: if *url* is not an http URL a shell word can carry. + :raises ValueError: if *url* is not an http(s) URL a shell word can carry. """ - if not PROXY_URL.match(url) or not url.startswith("http://"): + if not PROXY_URL.match(url): raise ValueError(f"not an http URL: {url!r}") options = ["-o Acquire::Retries=0", "-o Acquire::http::Timeout=10"] host = urlsplit(url).hostname or "" if direct and host and ":" not in host: - # *host* comes from a URL PROXY_URL let through: no quote in it. + # *host* comes from a URL PROXY_URL let through: no quote in it. For https + # the host's https setting comes first, so both are set. + if url.startswith("https://"): + options.append(f"-o 'Acquire::https::Proxy::{host}=DIRECT'") options.append(f"-o 'Acquire::http::Proxy::{host}=DIRECT'") match = f"echo '[match]'; grep -c -F -- {shlex.quote(expect)} \"$t\"; " if expect else "" return ( @@ -272,13 +364,16 @@ def parse_apt_fetch(output: str) -> ProxyFetchDict: return {"reachable": answered, "expected": False if answered else None, "error": error} -#: sed's edit of one line: comment it out, if it still sets the http proxy. -_COMMENT_OUT = "-e '{line}{{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}}'" +#: sed's edit of one line: comment it out, if it still sets the proxy of *scheme*. +_COMMENT_OUT = "-e '{line}{{/^[[:space:]]*Acquire::{scheme}::Proxy[[:space:]]/Is|^|// |}}'" +_SCHEMES = ("http", "https") -def comment_out_command(path: str, lines: list[int]) -> str: - """The root command that comments out *lines* of *path* where they set the http proxy.""" - edits = " ".join(_COMMENT_OUT.format(line=int(n)) for n in lines) +def comment_out_command(path: str, lines: list[int], scheme: str = "http") -> str: + """The root command that comments out *lines* of *path* where they set *scheme*'s proxy.""" + if scheme not in _SCHEMES: + raise ValueError(f"not a scheme apt proxies: {scheme!r}") + edits = " ".join(_COMMENT_OUT.format(line=int(n), scheme=scheme) for n in lines) return f"sed -i {edits} {shlex.quote(path)}" @@ -333,6 +428,8 @@ class AptProxyMixin(PackageProxyMixin): * per_host ({host: proxy or "DIRECT"}) * auto_detect (string or None): ``Acquire::http::Proxy-Auto-Detect`` * settings (list of {file, line, value, editable}), in apt's reading order + * https_proxy (string or None): the proxy https takes, the http one if unset + * https_per_host ({host: proxy or "DIRECT"}), https_settings (as settings) * http_sources, https_sources (lists of the sources' base URIs) :raises ValueError: if the host's output carried no intact report. @@ -360,8 +457,30 @@ class AptProxyMixin(PackageProxyMixin): self._run_apt_command(apt_fetch_command(url, direct=True, expect=expect)) ) + def check_https_source(self) -> HttpsSourceFetchDict | None: + """Fetch one https source's ``InRelease`` the way apt does, through its proxy. + + None when no https source goes through a proxy: there are none, or https + goes ``DIRECT``. + + :raises ValueError: if the host's output carried no intact report. + :raises NotImplementedError: on a host without apt. + """ + if not self._is_apt_host(): + raise NotImplementedError("not an apt host") + report = self._report() + found = _https_source(report, lambda proxy: proxy is not None) + if found is None: + return None + target, proxy = found + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + return {**fetched, "url": target, "proxy": proxy} + def configure_package_proxy(self, url: str) -> FixResultDict: - """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`.""" + """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`. + + https goes through *url* too, unless *url* refuses it: then ``DIRECT``. + """ if not url: return run_fix_steps([("configure", self._remove_package_proxy)]) return run_fix_steps( @@ -370,6 +489,7 @@ class AptProxyMixin(PackageProxyMixin): ("conflicts", lambda: self._comment_out_other_proxies(url)), ("verify", lambda: self._verify_package_proxy(url)), ("fetch", lambda: self._fetch_through_package_proxy(url)), + ("https", lambda: self._https_through_package_proxy(url)), ] ) @@ -383,12 +503,15 @@ class AptProxyMixin(PackageProxyMixin): f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" ) - def _write_package_proxy(self, url: str) -> FixStepOutcomeDict: + def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() if not PROXY_URL.match(url): raise ValueError(f"not a proxy URL: {url!r}") - return self._write_apt_file(self.APT_PROXY_FILE, f'Acquire::http::Proxy "{url}";\n') + content = f'Acquire::http::Proxy "{url}";\n' + if https_direct: + content += 'Acquire::https::Proxy "DIRECT";\n' + return self._write_apt_file(self.APT_PROXY_FILE, content) def _report(self) -> AptProxyReportDict: # The steps after "configure" run only once it passed, on an apt host. @@ -442,16 +565,81 @@ class AptProxyMixin(PackageProxyMixin): return {"ok": True, "output": f"fetched {target} through {url}"} return {"ok": False, "output": f"{target}: {fetched['error']}"} + def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict: + """An https source through *url*; where *url* refuses it, https goes ``DIRECT``.""" + found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url)) + if found is None: + return {"ok": True, "output": f"no https source goes through {url}; not tried"} + target = found[0] + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + # An HTTP error came from the mirror, through the tunnel: the cache let it by. + if fetched["reachable"]: + return {"ok": True, "output": f"fetched {target} through {url}"} + refused = f"{url} refused https ({fetched['error']})" + outcome = self._https_direct(url) + if not outcome["ok"]: + return {"ok": False, "output": f"{refused}; {outcome['output']}"} + direct = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if not direct["reachable"]: + return {"ok": False, "output": f"{refused}; direct too: {target}: {direct['error']}"} + named = f"; {outcome['output']}" if outcome["output"] else "" + return { + "ok": True, + "output": f"{refused}{named}; https sources go direct now: fetched {target}", + } + + def _https_direct(self, url: str) -> FixStepOutcomeDict: + """Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*.""" + written = self._write_package_proxy(url, https_direct=True) + if not written["ok"]: + return written + later = [ + s + for s in self._report()["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + for path, numbers in _editable_lines(later).items(): + outcome = self._run_apt_root_command( + comment_out_command(path, numbers, scheme="https"), f"commented out in {path}" + ) + if not outcome["ok"]: + return outcome + report = self._report() + still = [ + f"{s['file']}:{s['line']}" + for s in report["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + if same_proxy_url(report["https_proxy"], url): + where = f", set in {', '.join(still)}" if still else "" + return {"ok": False, "output": f"https still goes through {url}{where}"} + return {"ok": True, "output": "; ".join(map(_described, later))} + + +def _https_source( + report: AptProxyReportDict, through: Callable[[str | None], bool] +) -> tuple[str, str] | None: + """The ``InRelease`` of the first https source whose proxy *through* accepts, and that proxy.""" + for source in report["https_sources"]: + if "/dists/" not in source: + continue + proxy = https_proxy_for(report, urlsplit(source).hostname or "") + if through(proxy): + return source.rstrip("/") + "/InRelease", proxy or "" + return None + __all__ = [ "APT_PROXY_COMMAND", "AptProxyMixin", "AptProxyReportDict", "AptProxySettingDict", + "HttpsSourceFetchDict", "ProxyFetchDict", "PROXY_URL", "apt_fetch_command", "comment_out_command", + "https_proxy_for", "parse_apt_fetch", "parse_apt_proxy", "same_proxy_url", diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py index 3071f2d..b886211 100644 --- a/tests/test_apt_proxy.py +++ b/tests/test_apt_proxy.py @@ -20,6 +20,7 @@ from napalm_device_types.apt_proxy import ( AptProxyMixin, apt_fetch_command, comment_out_command, + https_proxy_for, parse_apt_fetch, parse_apt_proxy, same_proxy_url, @@ -141,7 +142,7 @@ class TestWhereItIsSet: class TestSources: - def test_https_sources_bypass_an_http_proxy(self): + def test_sources_by_scheme(self): report = parse_apt_proxy( _report( sources=[ @@ -155,6 +156,102 @@ class TestSources: assert report["https_sources"] == ["https://download.docker.com/linux/ubuntu/dists/noble/"] +class TestHttpsProxy: + """apt's https method asks ``Acquire::https::Proxy`` and then ``Acquire::http::Proxy``: + an http proxy alone takes the https sources too (NetOrk/netork#1811). Measured with + apt 2.8.3's apt-helper against a dead port.""" + + def test_the_http_proxy_alone_takes_https_too(self): + report = parse_apt_proxy(_report(dump=['Acquire::http::Proxy "http://c:3142";'])) + assert report["https_proxy"] == "http://c:3142" + + def test_an_https_proxy_of_its_own_wins(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy "http://other:3128";', + ] + ) + ) + + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", "http://other:3128") + + @pytest.mark.parametrize("value", ["DIRECT", "direct"]) + def test_direct_https_is_none(self, value): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://c:3142";', f'Acquire::https::Proxy "{value}";'] + ) + ) + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", None) + + def test_an_empty_https_proxy_is_not_set(self): + """apt takes an empty value for unset and goes on to the http one.""" + report = parse_apt_proxy( + _report(dump=['Acquire::http::Proxy "http://c:3142";', 'Acquire::https::Proxy "";']) + ) + assert report["https_proxy"] == "http://c:3142" + + def test_per_host_https_then_http(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy::a.example "DIRECT";', + 'Acquire::http::Proxy::b.example "DIRECT";', + 'Acquire::https::Proxy::b.example "http://c:3142";', + ] + ) + ) + + assert report["https_per_host"] == {"a.example": "DIRECT", "b.example": "http://c:3142"} + assert report["per_host"] == {"a.example": "DIRECT", "b.example": "DIRECT"} + + def test_where_the_https_proxy_is_set(self): + report = parse_apt_proxy( + _report( + parts=[ + f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";', + f'{_D}90curtin-aptproxy:2:Acquire::https::Proxy "http://c:3142";', + f'{_D}91x:1:Acquire::https {{ Proxy "http://n:1"; }};', + f'{_D}92x:1:// Acquire::https::Proxy "http://c:3142";', + f'{_D}93x:1:Acquire::https::Proxy::h.example "DIRECT";', + ] + ) + ) + + assert report["https_settings"] == [ + { + "file": f"{_D}90curtin-aptproxy", + "line": 2, + "value": "http://c:3142", + "editable": True, + }, + {"file": f"{_D}91x", "line": 1, "value": "http://n:1", "editable": False}, + ] + assert [s["file"] for s in report["settings"]] == [f"{_D}00proxy"] + + @pytest.mark.parametrize( + "host, proxy", + [ + ("deb.example", "http://c:3142"), + ("direct.example", None), + ("Other.Example", "http://o:1"), + ], + ) + def test_the_proxy_for_one_host(self, host, proxy): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy::direct.example "DIRECT";', + 'Acquire::http::Proxy::other.example "http://o:1";', + ] + ) + ) + assert https_proxy_for(report, host) == proxy + + class TestSameProxyUrl: @pytest.mark.parametrize( "a, b", @@ -199,6 +296,12 @@ E: Download Failed""" _FETCHED = """Get:1 http://172.22.8.50:3142/acng-report.html Fetched 14.0 kB in 0s (0 B/s)""" +#: apt-cacher-ng without a PassThroughPattern, as apt 2.8.3 tells it (NetOrk/netork#1811). +_CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/InRelease + Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Download Failed""" + def _fetch(output, rc, match=None): lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] @@ -243,6 +346,12 @@ class TestFetch: assert result["expected"] is False assert result["error"] == "503 Host not found [IP: 172.22.8.50 3142]" + def test_a_refused_tunnel(self): + result = parse_apt_fetch(_fetch(_CONNECT_DENIED, 100)) + + assert result["reachable"] is False + assert "403 CONNECT denied" in result["error"] + def test_without_apt_helper_nothing_could_be_tried(self): with pytest.raises(ValueError, match="apt-helper"): parse_apt_fetch(_fetch("sh: 1: /usr/lib/apt/apt-helper: not found", 127)) @@ -265,6 +374,21 @@ class TestFetchCommand: command = apt_fetch_command("http://archive.ubuntu.com/ubuntu/dists/noble/InRelease") assert "DIRECT" not in command + def test_an_https_source_through_apt_s_own_proxy(self): + command = apt_fetch_command( + "https://download.docker.com/linux/ubuntu/dists/noble/InRelease" + ) + assert "DIRECT" not in command + assert "download-file https://download.docker.com/" in command + + def test_past_the_proxy_to_an_https_host(self): + """For https the host's own https setting comes first; the http one alone + would lose to an https proxy for that host.""" + command = apt_fetch_command("https://h.example/x", direct=True) + + assert "'Acquire::https::Proxy::h.example=DIRECT'" in command + assert "'Acquire::http::Proxy::h.example=DIRECT'" in command + @pytest.mark.parametrize("url", ["http://c:3142/x'; rm -rf /", "ftp://c/x", "c:3142"]) def test_a_url_that_would_break_the_command_is_refused(self, url): with pytest.raises(ValueError): @@ -275,6 +399,7 @@ def test_the_commands_are_valid_posix_sh(): for command in ( APT_PROXY_COMMAND, apt_fetch_command("http://c:3142/x", direct=True, expect="y"), + apt_fetch_command("https://h.example/dists/x/InRelease"), ): subprocess.run(["sh", "-n", "-c", command], check=True) @@ -283,16 +408,28 @@ def test_the_commands_are_valid_posix_sh(): def test_the_report_command_on_this_host(): output = subprocess.run(["sh", "-c", APT_PROXY_COMMAND], capture_output=True, text=True).stdout report = parse_apt_proxy(output) - assert set(report) >= {"proxy", "settings", "https_sources"} + assert set(report) >= {"proxy", "https_proxy", "settings", "https_settings", "https_sources"} + + +_DOCKER = "https://download.docker.com/linux/ubuntu/dists/noble/" class _Host(AptProxyMixin): - """A Debian host: its apt files, and what apt-config would make of them.""" + """A Debian host: its apt files, and what apt-config would make of them. - def __init__(self, files=None, fetch_rc=0, apt=True): + *https_sources* are its sources over https. *tunnel* is whether the proxy lets + them through; *direct_rc* is how a fetch of one goes that takes no proxy. + """ + + def __init__( + self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0 + ): self.files = dict(files or {}) self.fetch_rc = fetch_rc self.apt = apt + self.https_sources = list(https_sources) + self.tunnel = tunnel + self.direct_rc = direct_rc self.root = [] self.fetched = [] @@ -309,26 +446,36 @@ class _Host(AptProxyMixin): out.append((path, n, text)) return out - def _effective(self): + def _effective(self, scheme="http"): value = None for _path, _n, text in self._settings(): - m = re.match(r'\s*Acquire::http::Proxy\s+"([^"]*)"', text, re.I) + m = re.match(rf'\s*Acquire::{scheme}::Proxy\s+"([^"]*)"', text, re.I) if m: value = m.group(1) return value def _run_apt_command(self, command): if command == APT_PROXY_COMMAND: - value = self._effective() + value, https = self._effective(), self._effective("https") + dump = [f'Acquire::http::Proxy "{value}";'] if value else [] + dump += [f'Acquire::https::Proxy "{https}";'] if https else [] return _report( - dump=[f'Acquire::http::Proxy "{value}";'] if value else [], + dump=dump, parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], main=[f"{p}:{n}:{t}" for p, n, t in self._settings() if not p.startswith(_D)], - sources=["http://deb.debian.org/debian/dists/trixie/"], + sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources], ) self.fetched.append(command) + if re.search(r"download-file '?https://", command): + return self._fetch_https(command) return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) + def _fetch_https(self, command): + https = self._effective("https") or self._effective() + if "DIRECT" in command or not https or https.upper() == "DIRECT": + return _fetch(_FETCHED if self.direct_rc == 0 else _UNRESOLVED, self.direct_rc) + return _fetch(_FETCHED, 0) if self.tunnel else _fetch(_CONNECT_DENIED, 100) + def _write_apt_file(self, path, content): self.files[path] = content return {"ok": True, "output": f"wrote {path}"} @@ -346,6 +493,41 @@ class _Host(AptProxyMixin): return {"ok": True, "output": done} +class TestCheckHttpsSource: + def test_one_source_the_way_apt_fetches_it(self): + host = _Host( + {f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.check_https_source() + + assert result["url"] == f"{_DOCKER}InRelease" + assert result["proxy"] == "http://c:3142" + assert result["reachable"] is False and "403 CONNECT denied" in result["error"] + assert "DIRECT" not in host.fetched[0] + + def test_none_without_an_https_source(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + assert host.check_https_source() is None + + def test_none_where_https_goes_direct(self): + host = _Host( + { + f"{_D}00proxy": ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + }, + https_sources=[_DOCKER], + ) + assert host.check_https_source() is None + + def test_not_on_a_host_without_apt(self): + with pytest.raises(NotImplementedError): + _Host(apt=False).check_https_source() + + class TestTheMixin: def test_it_is_a_package_proxy(self): assert issubclass(AptProxyMixin, PackageProxyMixin) @@ -371,6 +553,10 @@ class TestTheMixin: assert "Proxy::c=DIRECT" in host.fetched[0] +def _step(result, name): + return next(s for s in result["steps"] if s["step"] == name) + + class TestConfigurePackageProxy: def test_a_fresh_host(self): host = _Host() @@ -378,7 +564,14 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") assert result["success"] is True - assert [s["step"] for s in result["steps"]] == ["configure", "conflicts", "verify", "fetch"] + assert [s["step"] for s in result["steps"]] == [ + "configure", + "conflicts", + "verify", + "fetch", + "https", + ] + assert "no https source" in result["steps"][-1]["output"] assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] @@ -410,7 +603,7 @@ class TestConfigurePackageProxy: def test_a_setting_it_cannot_comment_out_fails_verification(self): host = _Host({"/etc/apt/apt.conf": 'Acquire::http { Proxy "http://older:8080"; };\n'}) - host._effective = lambda: "http://older:8080" + host._effective = lambda scheme="http": "http://older:8080" result = host.configure_package_proxy("http://c:3142") @@ -428,6 +621,100 @@ class TestConfigurePackageProxy: assert result["steps"][-1]["step"] == "fetch" assert "Connection refused" in result["steps"][-1]["output"] + def test_an_https_source_the_cache_tunnels_stays_with_it(self): + host = _Host(https_sources=[_DOCKER]) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' + assert result["steps"][-1]["output"] == ( + f"fetched {_DOCKER}InRelease through http://c:3142" + ) + + def test_an_https_source_the_cache_refuses_goes_direct(self): + """NetOrk/netork#1811: apt-cacher-ng without a PassThroughPattern answers + 403 to every https source apt sends it.""" + host = _Host(https_sources=[_DOCKER], tunnel=False) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + output = result["steps"][-1]["output"] + assert "http://c:3142 refused https" in output and "403 CONNECT denied" in output + assert f"https sources go direct now: fetched {_DOCKER}InRelease" in output + + def test_a_later_https_line_to_the_cache_is_commented_out(self): + """Ubuntu's installer writes the https proxy too; it would win over 00proxy.""" + curtin = f"{_D}90curtin-aptproxy" + host = _Host( + { + curtin: ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Acquire::https::Proxy "http://c:3142";\n' + ) + }, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[curtin] == ( + 'Acquire::http::Proxy "http://c:3142";\n// Acquire::https::Proxy "http://c:3142";\n' + ) + assert f"{curtin}:2 (http://c:3142) commented out" in result["steps"][-1]["output"] + + def test_an_https_proxy_of_its_own_is_not_the_fix_s(self): + host = _Host( + {f"{_D}01https": 'Acquire::https::Proxy "http://corp:3128";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "no https source goes through http://c:3142" in result["steps"][-1]["output"] + assert f"{_D}00proxy" in host.files and "https" not in host.files[f"{_D}00proxy"] + + def test_an_http_error_through_the_tunnel_is_a_tunnel(self): + """The mirror answered, so the cache let the source through.""" + host = _Host(https_sources=[_DOCKER]) + host._fetch_https = lambda command: _fetch(_HTTP_ERROR, 100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "https" not in host.files[f"{_D}00proxy"] + + def test_refused_and_no_way_direct_fails(self): + host = _Host(https_sources=[_DOCKER], tunnel=False, direct_rc=100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "Could not resolve" in result["steps"][-1]["output"] + + def test_a_later_https_line_it_cannot_comment_out_fails(self): + host = _Host( + {"/etc/apt/apt.conf": 'Acquire::https { Proxy "http://c:3142"; };\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + host._effective = lambda scheme="http": "http://c:3142" + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"] + def test_without_an_http_source_the_fetch_says_so(self): host = _Host() real = host._run_apt_command @@ -440,7 +727,7 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") assert result["success"] is True - assert "no http source" in result["steps"][-1]["output"] + assert "no http source" in _step(result, "fetch")["output"] def test_a_direct_source_is_not_proof(self): host = _Host() @@ -451,7 +738,7 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") - assert "no http source" in result["steps"][-1]["output"] + assert "no http source" in _step(result, "fetch")["output"] @pytest.mark.parametrize("url", ['http://x"; Acquire::x "y', "ftp://c", "c:3142"]) def test_a_url_that_would_break_the_file_is_refused(self, url): @@ -531,3 +818,9 @@ def test_the_comment_out_command(): "sed -i -e '2{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' " "-e '4{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" ) + + +def test_the_comment_out_command_for_https(): + assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == ( + "sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" + ) From c71ddd7fe6c7f48de4df7517e41a3a09f9a57656 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 08:13:20 +0100 Subject: [PATCH 34/50] refactor(apt_proxy): smaller parts, under the standard's complexity limit The quality ratchet counted _dump, parse_apt_proxy and _https_direct over complexity 10 (NetOrk/netork#1811). Same behaviour, same tests. --- napalm_device_types/apt_proxy.py | 73 ++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index e8e933a..daa70c3 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -226,22 +226,31 @@ def _dump(lines: list[str]) -> _Dump: "https_per_host": {}, } for found in (_DUMP.match(line.strip()) for line in lines): - if not found: - continue - scheme, suffix, value = found.groups() - https = scheme.lower() == "https" - if not suffix and https: - dump["https"] = value or None - elif not suffix: - dump["proxy"] = None if value.lower() in _NO_PROXY else value - elif suffix.lower() == "-auto-detect": - if not https: - dump["auto_detect"] = value or None - elif value: - dump["https_per_host" if https else "per_host"][suffix[2:]] = value + if found: + _take(dump, *found.groups()) return dump +def _take(dump: _Dump, scheme: str, suffix: str | None, value: str) -> None: + """One line of the dump into *dump*; a later one of the same key wins, as in apt.""" + https = scheme.lower() == "https" + if not suffix: + _take_for_all(dump, https, value) + elif suffix.lower() == "-auto-detect": + if not https: + dump["auto_detect"] = value or None + elif value: + dump["https_per_host" if https else "per_host"][suffix[2:]] = value + + +def _take_for_all(dump: _Dump, https: bool, value: str) -> None: + """The proxy for every host: https keeps an empty value as unset, apt's fallback.""" + if https: + dump["https"] = value or None + else: + dump["proxy"] = None if value.lower() in _NO_PROXY else value + + def _https_proxy(dump: _Dump) -> str | None: """apt's https method takes its own setting and falls back to the http one.""" if dump["https"] is None: @@ -278,15 +287,25 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: "proxy": dump["proxy"], "per_host": dump["per_host"], "auto_detect": dump["auto_detect"], - "settings": [s for s in (_setting(line, main) for main, line in lines) if s], + "settings": _settings(lines, _setting), "https_proxy": _https_proxy(dump), "https_per_host": {**dump["per_host"], **dump["https_per_host"]}, - "https_settings": [s for s in (_https_setting(line, main) for main, line in lines) if s], - "http_sources": [s for s in sources if s.startswith("http://")], - "https_sources": [s for s in sources if s.startswith("https://")], + "https_settings": _settings(lines, _https_setting), + "http_sources": _of_scheme(sources, "http"), + "https_sources": _of_scheme(sources, "https"), } +def _settings( + lines: list[tuple[bool, str]], parse: Callable[[str, bool], AptProxySettingDict | None] +) -> list[AptProxySettingDict]: + return [s for s in (parse(line, main) for main, line in lines) if s] + + +def _of_scheme(sources: list[str], scheme: str) -> list[str]: + return [s for s in sources if s.startswith(f"{scheme}://")] + + _DEFAULT_PORT = {"http": 80, "https": 443} @@ -593,11 +612,7 @@ class AptProxyMixin(PackageProxyMixin): written = self._write_package_proxy(url, https_direct=True) if not written["ok"]: return written - later = [ - s - for s in self._report()["https_settings"] - if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) - ] + later = self._later_https_lines(self._report(), url) for path, numbers in _editable_lines(later).items(): outcome = self._run_apt_root_command( comment_out_command(path, numbers, scheme="https"), f"commented out in {path}" @@ -605,16 +620,20 @@ class AptProxyMixin(PackageProxyMixin): if not outcome["ok"]: return outcome report = self._report() - still = [ - f"{s['file']}:{s['line']}" - for s in report["https_settings"] - if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) - ] if same_proxy_url(report["https_proxy"], url): + still = [f"{s['file']}:{s['line']}" for s in self._later_https_lines(report, url)] where = f", set in {', '.join(still)}" if still else "" return {"ok": False, "output": f"https still goes through {url}{where}"} return {"ok": True, "output": "; ".join(map(_described, later))} + def _later_https_lines(self, report: AptProxyReportDict, url: str) -> list[AptProxySettingDict]: + """The lines outside :attr:`APT_PROXY_FILE` that send https to *url*.""" + return [ + s + for s in report["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + def _https_source( report: AptProxyReportDict, through: Callable[[str | None], bool] From 449a6b13ca508641275a7aca77e4e38fc32745ff Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 08:17:27 +0100 Subject: [PATCH 35/50] refactor(ssh_server): split three functions under the standard's line limit (#65) harden_ssh_server (70), _apply_or_restore (51) and exempt_from_ssh_penalties (81) had grown past the 50-line limit with #63, and `driver / standard` was red on main. The reading and planning, the putting back, the session's address, the reasons to leave the exemption and its write move into methods of their own. Same behaviour, same tests; harden_ssh_server leaves the baseline. --- napalm_device_types/ssh_server.py | 192 ++++++++++++++++++------------ tests/quality_baseline.json | 1 - 2 files changed, 115 insertions(+), 78 deletions(-) diff --git a/napalm_device_types/ssh_server.py b/napalm_device_types/ssh_server.py index 8a67d55..6c750bf 100644 --- a/napalm_device_types/ssh_server.py +++ b/napalm_device_types/ssh_server.py @@ -402,6 +402,14 @@ class SshPenaltyExemption: message: str = "" +#: Why an algorithm stays, by :class:`Refusal` reason. +_WHYS = { + "last": "it is the last one", + "client": "netOrk could no longer connect", + "host_keys_unknown": "the host keys could not be read", +} + + def _unchanged( result: SshPenaltyExemption, message: str, say: Callable[[str], None] ) -> SshPenaltyExemption: @@ -544,24 +552,7 @@ class OpenSshServerMixin: :raises RuntimeError: if the current state cannot be read. """ say = log or (lambda _line: None) - say("Reading the algorithms sshd offers (sshd -T)") - sshd_t = self._sshd_effective_text() - host_keys: set[str] | None = None - if remove.get("hostkey"): - say("Reading which host keys the server has (ssh-keygen -l)") - host_keys = self._ssh_host_key_algorithms(sshd_t) - plan = plan_ssh_hardening( - parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys - ) - whys = { - "last": "it is the last one", - "client": "netOrk could no longer connect", - "host_keys_unknown": "the host keys could not be read", - } - for r in plan.refused: - last_key = r.reason == "last" and r.kind == "hostkey" - why = "it is the last one the host has a key for" if last_key else whys[r.reason] - say(f"Keeping {r.algorithm}: {why}") + plan = self._plan_hardening(remove, client, say) kinds = plan.changed_classes result = SshHardeningResult(plan=plan) if not kinds: @@ -575,26 +566,50 @@ class OpenSshServerMixin: if main is None: raise RuntimeError(f"cannot read {MAIN_CONFIG}") restore = self._write_change(plan, main, result, say) - - def ignored() -> str | None: - now = self.get_ssh_server_algorithms() - names = [SSH_ALGORITHM_CLASSES[k][1] for k in kinds if now[k] != plan.keep[k]] - if not names: - return None - return ( - f"sshd does not use the new {', '.join(names)}: a setting read before" - f" {result.path} decides them" - ) - return self._apply_or_restore( result, restore, verify, say, - ignored=ignored, + ignored=lambda: self._algorithms_ignored(plan, result.path), done="The server no longer offers the selected algorithms.", ) + def _plan_hardening( + self, + remove: Mapping[str, Iterable[str]], + client: Mapping[str, Iterable[str]] | None, + say: Callable[[str], None], + ) -> SshHardeningPlan: + """Read what sshd offers and which host keys it has; plan, and say what stays.""" + say("Reading the algorithms sshd offers (sshd -T)") + sshd_t = self._sshd_effective_text() + host_keys: set[str] | None = None + if remove.get("hostkey"): + say("Reading which host keys the server has (ssh-keygen -l)") + host_keys = self._ssh_host_key_algorithms(sshd_t) + plan = plan_ssh_hardening( + parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys + ) + for r in plan.refused: + last_key = r.reason == "last" and r.kind == "hostkey" + why = "it is the last one the host has a key for" if last_key else _WHYS[r.reason] + say(f"Keeping {r.algorithm}: {why}") + return plan + + def _algorithms_ignored(self, plan: SshHardeningPlan, path: str | None) -> str | None: + """Why sshd does not use the new lists; None when it does.""" + now = self.get_ssh_server_algorithms() + names = [ + SSH_ALGORITHM_CLASSES[k][1] for k in plan.changed_classes if now[k] != plan.keep[k] + ] + if not names: + return None + return ( + f"sshd does not use the new {', '.join(names)}: a setting read before" + f" {path} decides them" + ) + def _write_change( self, plan: SshHardeningPlan, @@ -646,19 +661,7 @@ class OpenSshServerMixin: """ def back(message: str, reload: bool) -> R: - say(message) - say("Putting the previous configuration back") - try: - restore() - if reload: - self._reload_ssh_server() - except Exception as exc: # noqa: BLE001 - report it, the caller must look - result.message = f"{message}; putting the previous configuration back failed: {exc}" - say(result.message) - return result - result.rolled_back = True - result.message = message - return result + return self._put_back(result, restore, say, message, reload=reload) try: say("Checking the configuration (sshd -t)") @@ -681,6 +684,30 @@ class OpenSshServerMixin: say(result.message) return result + def _put_back[R: (SshHardeningResult, SshPenaltyExemption)]( + self, + result: R, + restore: Callable[[], None], + say: Callable[[str], None], + message: str, + *, + reload: bool, + ) -> R: + """Say *message*, put the previous state back, and say whether that worked.""" + say(message) + say("Putting the previous configuration back") + try: + restore() + if reload: + self._reload_ssh_server() + except Exception as exc: # noqa: BLE001 - report it, the caller must look + result.message = f"{message}; putting the previous configuration back failed: {exc}" + say(result.message) + return result + result.rolled_back = True + result.message = message + return result + # ── the penalty exemption ─────────────────────────────────────────── def exempt_from_ssh_penalties( @@ -708,39 +735,65 @@ class OpenSshServerMixin: ``sshd -T`` or sshd_config cannot be read. """ say = log or (lambda _line: None) - result = SshPenaltyExemption() + address = self._session_client_address() + result = SshPenaltyExemption(address=address) + say(f"netOrk connects from {address}") + say("Reading sshd's per-source penalties (sshd -T)") + exempt = parse_penalty_exemptions(self._sshd_effective_text()) + reason = self._penalty_exemption_unchanged(address, exempt) + if reason is not None or exempt is None: + return _unchanged(result, reason or "", say) + + restore = self._write_penalty_dropin([*exempt, address]) + result.path = PENALTY_DROPIN_PATH + say(f"Wrote {PENALTY_DROPIN_PATH}") + return self._apply_or_restore( + result, + restore, + verify, + say, + ignored=lambda: self._exemption_ignored(address), + done=f"{address} is exempt from sshd's per-source penalties.", + ) + + def _session_client_address(self) -> str: + """The address the host sees this session come from (``$SSH_CONNECTION``).""" session = self.run_command('printf %s "$SSH_CONNECTION"', privileged=False, timeout=30) address = client_address(session.stdout) if address is None: raise RuntimeError("the session names no client address ($SSH_CONNECTION)") - result.address = address - say(f"netOrk connects from {address}") - say("Reading sshd's per-source penalties (sshd -T)") - exempt = parse_penalty_exemptions(self._sshd_effective_text()) + return address + + def _penalty_exemption_unchanged(self, address: str, exempt: list[str] | None) -> str | None: + """Why the exemption is left as it is; None when it is to be written.""" if exempt is None: - return _unchanged( - result, - "Nothing to change: this sshd has no per-source penalties (before OpenSSH 9.8).", - say, - ) + return "Nothing to change: this sshd has no per-source penalties (before OpenSSH 9.8)." if is_exempt(address, exempt): - return _unchanged(result, f"Nothing to change: {address} is already exempt.", say) + return f"Nothing to change: {address} is already exempt." main = self._ssh_read(MAIN_CONFIG) if main is None: raise RuntimeError(f"cannot read {MAIN_CONFIG}") if not has_dropin_include(main): - return _unchanged( - result, + return ( f"Not changed: {MAIN_CONFIG} does not include {DROPIN_DIR}," - " where the exemption would go.", - say, + " where the exemption would go." ) + return None + def _exemption_ignored(self, address: str) -> str | None: + """Why sshd does not use the new exemption; None when it does.""" + if is_exempt(address, parse_penalty_exemptions(self._sshd_effective_text()) or []): + return None + return ( + "sshd does not use the new PerSourcePenaltyExemptList: a setting read before" + f" {PENALTY_DROPIN_PATH} decides it" + ) + + def _write_penalty_dropin(self, exempt: Sequence[str]) -> Callable[[], None]: + """Write the exemption list; return what puts the previous drop-in back.""" previous = self._ssh_read(PENALTY_DROPIN_PATH) self._ssh_run(f"mkdir -p -- {quote(DROPIN_DIR)}") - self._ssh_write(PENALTY_DROPIN_PATH, render_penalty_dropin([*exempt, address])) - result.path = PENALTY_DROPIN_PATH - say(f"Wrote {PENALTY_DROPIN_PATH}") + self._ssh_write(PENALTY_DROPIN_PATH, render_penalty_dropin(exempt)) def restore() -> None: if previous is None: @@ -748,19 +801,4 @@ class OpenSshServerMixin: else: self._ssh_write(PENALTY_DROPIN_PATH, previous) - def ignored() -> str | None: - if is_exempt(address, parse_penalty_exemptions(self._sshd_effective_text()) or []): - return None - return ( - "sshd does not use the new PerSourcePenaltyExemptList: a setting read before" - f" {PENALTY_DROPIN_PATH} decides it" - ) - - return self._apply_or_restore( - result, - restore, - verify, - say, - ignored=ignored, - done=f"{address} is exempt from sshd's per-source penalties.", - ) + return restore diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 488f4d5..82c402c 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -25,7 +25,6 @@ "napalm_device_types/firewall_rules.py::FirewallRuleMixin.diff_firewall_rules": 56, "napalm_device_types/hypervisor.py::HypervisorDriver.create_vm_from_cloud_init": 84, "napalm_device_types/hypervisor.py::HypervisorDriver.get_vm_config": 77, - "napalm_device_types/ssh_server.py::OpenSshServerMixin.harden_ssh_server": 52, "napalm_device_types/ssh_server.py::plan_ssh_hardening": 60, "napalm_device_types/storage.py::StorageDriver.get_volumes": 52, "napalm_device_types/switch.py::SwitchDriver.get_dot1x_ports": 51, From 666be091e36df8b24f231a24319066dc38579f58 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 12:03:01 +0100 Subject: [PATCH 36/50] feat: netOrk's own account through the current login (NetOrk/netork#1817) add_service_account(name, public_key) creates or reopens the account through the login the driver uses today: a key and no password, and a rule of its own in 85-service- that lets it run everything as root without a password, checked with visudo and sudo -l. Root is refused; run again with the account the driver already logs in with, it only adds what is missing. --- CHANGELOG.md | 6 + napalm_device_types/local_accounts.py | 124 ++++++++++++++-- tests/test_service_account.py | 199 ++++++++++++++++++++++++++ 3 files changed, 315 insertions(+), 14 deletions(-) create mode 100644 tests/test_service_account.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3312e3f..6f1c27e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `LocalAccountsMixin.add_service_account(name, public_key)`: netOrk's own account, + created through the account the driver logs in with today. It takes a key and no + password, and a rule of its own in `85-service-` lets it run everything as + root without a password (`NOPASSWD: ALL`, checked with `visudo` and `sudo -l`). An + account that is there is opened again and keeps its other keys; root is refused + (NetOrk/netork#1817). - `LocalAccountsMixin` (`napalm_device_types.local_accounts`): personal login accounts on a host. `get_local_accounts(names)` reads them back (groups, lock, sudo, key files), `add_local_account(name, public_key, password_hash=, admin=)` creates or diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py index d579bc7..9cde5c3 100644 --- a/napalm_device_types/local_accounts.py +++ b/napalm_device_types/local_accounts.py @@ -31,6 +31,12 @@ afterwards shows whether it took effect. A rule elsewhere -- cloud-init's **The login user's own account and system accounts are never touched**: locking the first would lock the caller out. + +**netOrk's own account** (``add_service_account``) is made the same way, +through the account it logs in with today: a key and no password, and a rule +of its own -- `` ALL=(ALL) NOPASSWD: ALL`` in ``85-service-`` -- +so it gains root without a password that could end up anywhere. Once netOrk +logs in with it, the person's account it used before is one like any other. """ from __future__ import annotations @@ -68,9 +74,16 @@ _KEY_BLOB_RE = re.compile(r"[A-Za-z0-9+/]+={0,3}") #: Before the rule's file name; after cloud-init's ``90-`` it would win, so before it. SUDOERS_FILE_PREFIX = "80-account-" +#: Before the file name of netOrk's own account's rule; after a person's, before cloud-init's. +SERVICE_SUDOERS_FILE_PREFIX = "85-service-" #: A line of ``sudo -l`` that lets the account run every command as root. _RUN_ALL_RE = re.compile(r"^\s*\(ALL(\s*:\s*ALL)?\)\s+([A-Z_]+:\s*)*ALL\s*$", re.MULTILINE) +#: The same, without a password. +_RUN_ALL_NOPASSWD_RE = re.compile( + r"^\s*\(ALL(\s*:\s*ALL)?\)\s+([A-Z_]+:\s*)*NOPASSWD:\s*([A-Z_]+:\s*)*ALL\s*$", + re.MULTILINE, +) #: Stages stdin as a file anyone may read -- the person reads the key file #: from it -- and prints its path. The content is a public key or a sudo rule. @@ -114,8 +127,21 @@ def sudoers_rule_path(directory: str, name: str) -> str: sudo skips a file whose name holds a dot, so a dot becomes ``_2e``, and an underscore ``__``: no two names share a file. """ - escaped = name.replace("_", "__").replace(".", "_2e") - return f"{directory}/{SUDOERS_FILE_PREFIX}{escaped}" + return f"{directory}/{SUDOERS_FILE_PREFIX}{_rule_file_name(name)}" + + +def service_sudoers_rule(name: str) -> str: + """The rule that lets netOrk's own account *name* run everything as root, no password.""" + return f"{name} ALL=(ALL) NOPASSWD: ALL\n" + + +def service_sudoers_rule_path(directory: str, name: str) -> str: + """Where the rule of netOrk's own account *name* lives in *directory*.""" + return f"{directory}/{SERVICE_SUDOERS_FILE_PREFIX}{_rule_file_name(name)}" + + +def _rule_file_name(name: str) -> str: + return name.replace("_", "__").replace(".", "_2e") def authorized_keys_paths(home: str) -> tuple[str, str]: @@ -357,6 +383,46 @@ class LocalAccountsMixin: ] return self._accounts_result(name, parts) + def add_service_account(self, name: str, public_key: str) -> LocalAccountChangeDict: + """Make *name* netOrk's own account: it logs in with *public_key*, and gains + root without a password. + + Created through the account the driver logs in with now, or opened + again when it is there; its other keys stay. It takes no password. Run + again with the account the driver already logs in with, it only adds + what is missing. + + Steps as in :meth:`add_local_account`: ``account``, then ``sudo`` and + ``keys``. The new login itself is for the caller to try. + + :raises ValueError: for a name or key no host takes, or root. + :raises RuntimeError: when the login user cannot be told. + """ + check_account_name(name) + check_public_key(public_key) + login = self._accounts_login() + existing = self._accounts_passwd(name) + if existing is not None and int(existing[2]) == 0: + raise ValueError(f"{name} is root") + if name == login: + step: FixStepDict = { + "step": "account", + "ok": True, + "output": f"this driver logs in as {name} already", + } + first: FixResultDict = _joined([step]) + else: + first = run_fix_steps( + [("account", lambda: self._accounts_open_or_create(name, existing, None))] + ) + parts = [first] + if first["success"]: + parts += [ + run_fix_steps([("sudo", lambda: self._accounts_grant_service_sudo(name))]), + run_fix_steps([("keys", lambda: self._accounts_add_key(name, public_key))]), + ] + return self._accounts_result(name, parts) + def _accounts_check_hash(self, password_hash: str) -> None: prefixes = PASSWORD_SCHEMES[self.ACCOUNT_PASSWORD_SCHEME] if not _CRYPT_RE.fullmatch(password_hash) or not password_hash.startswith(prefixes): @@ -368,16 +434,20 @@ class LocalAccountsMixin: :raises ValueError: for the login user's own account or a system account. :raises RuntimeError: when the login user cannot be told. """ - login = self.run_command("id -un", timeout=30).stdout.strip() - if not login: - raise RuntimeError("cannot tell which account this driver logs in with") - if name == login: + if name == self._accounts_login(): raise ValueError(f"{name} is the account this driver logs in with") passwd = self._accounts_passwd(name) if passwd is not None and int(passwd[2]) < self.ACCOUNT_MIN_UID: raise ValueError(f"{name} is a system account (uid {passwd[2]})") return passwd + def _accounts_login(self) -> str: + """:raises RuntimeError: when the login user cannot be told.""" + login = self.run_command("id -un", timeout=30).stdout.strip() + if not login: + raise RuntimeError("cannot tell which account this driver logs in with") + return login + def _accounts_result(self, name: str, parts: list[FixResultDict]) -> LocalAccountChangeDict: steps = [s for part in parts for s in part["steps"]] result = _joined(steps) @@ -455,7 +525,36 @@ class LocalAccountsMixin: return {"ok": True, "output": "\n".join(said)} def _accounts_grant_sudo(self, name: str, path: str) -> FixStepOutcomeDict: - staged = self._accounts_stage(sudoers_rule(name)) + failed = self._accounts_install_rule(sudoers_rule(name), path) + if failed is not None: + return failed + if not self._accounts_admin(name): + return { + "ok": False, + "output": f"{path} is written, but sudo does not read it: " + f"does sudoers include {self.SUDOERS_DIR}?", + } + return {"ok": True, "output": f"{name} may use sudo with its password ({path})"} + + def _accounts_grant_service_sudo(self, name: str) -> FixStepOutcomeDict: + path = service_sudoers_rule_path(self.SUDOERS_DIR, name) + failed = self._accounts_install_rule(service_sudoers_rule(name), path) + if failed is not None: + return failed + result = self.run_command(_command(["sudo", "-l", "-U", name]), privileged=True, timeout=30) + if result.exit_code != 0: + return {"ok": False, "output": f"cannot ask sudo about {name}: {_said(result)}"} + if not _RUN_ALL_NOPASSWD_RE.search(result.stdout): + return { + "ok": False, + "output": f"{path} is written, but sudo does not read it: " + f"does sudoers include {self.SUDOERS_DIR}?", + } + return {"ok": True, "output": f"{name} may use sudo without a password ({path})"} + + def _accounts_install_rule(self, rule: str, path: str) -> FixStepOutcomeDict | None: + """Check *rule* with visudo and install it at *path*; the failure, or None.""" + staged = self._accounts_stage(rule) try: checked = self._accounts_root(["visudo", "-cf", staged]) if checked.exit_code != 0: @@ -467,13 +566,7 @@ class LocalAccountsMixin: return {"ok": False, "output": f"{path}: {_said(installed)}"} finally: self._accounts_unstage(staged) - if not self._accounts_admin(name): - return { - "ok": False, - "output": f"{path} is written, but sudo does not read it: " - f"does sudoers include {self.SUDOERS_DIR}?", - } - return {"ok": True, "output": f"{name} may use sudo with its password ({path})"} + return None # -- keys -------------------------------------------------------------------------- @@ -576,6 +669,7 @@ def _joined(steps: list[FixStepDict]) -> FixResultDict: __all__ = [ "ACCOUNT_NAME_RE", "PASSWORD_SCHEMES", + "SERVICE_SUDOERS_FILE_PREFIX", "SUDOERS_FILE_PREFIX", "LocalAccountChangeDict", "LocalAccountDict", @@ -583,6 +677,8 @@ __all__ = [ "authorized_keys_paths", "check_account_name", "check_public_key", + "service_sudoers_rule", + "service_sudoers_rule_path", "sudoers_rule", "sudoers_rule_path", ] diff --git a/tests/test_service_account.py b/tests/test_service_account.py new file mode 100644 index 0000000..0c53bb7 --- /dev/null +++ b/tests/test_service_account.py @@ -0,0 +1,199 @@ +"""netOrk's own account on a host: created through the account it logs in with today. + +The fake host of :mod:`tests.test_local_accounts`, taught the rule that lets +an account run everything as root without a password. +""" + +from __future__ import annotations + +import shlex + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.local_accounts import ( + service_sudoers_rule, + service_sudoers_rule_path, + sudoers_rule_path, +) +from tests.test_local_accounts import ( + OWN_KEY, + SHELL_SYNTAX, + _Account, + _Driver, + _Host, + _root_commands, +) + +NETORK_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINetork netork@central" + + +class _ServiceHost(_Host): + """The fake host, whose sudo also takes the rule without a password.""" + + def is_passwordless(self, name: str) -> bool: + rule = self.files.get(service_sudoers_rule_path(_Driver.SUDOERS_DIR, name)) + return self.includes_dropins and rule is not None + + def _sudo(self, args: list[str], privileged: bool) -> CommandResult: + name = args[2] + if not self.is_passwordless(name): + return super()._sudo(args, privileged) + out = f"User {name} may run the following commands on host:\n (ALL) NOPASSWD: ALL\n" + return CommandResult(out, "", 0) + + def _visudo(self, args: list[str], privileged: bool) -> CommandResult: + content = self.files[args[1]] + if content.endswith(" ALL=(ALL) NOPASSWD: ALL\n") and "\n" not in content[:-1]: + return CommandResult("parsed OK", "", 0) + return super()._visudo(args, privileged) + + +@pytest.fixture +def host() -> _ServiceHost: + return _ServiceHost() + + +@pytest.fixture +def driver(host: _ServiceHost) -> _Driver: + return _Driver(host) + + +def test_the_rule_runs_everything_as_root_without_a_password() -> None: + assert service_sudoers_rule("netork") == "netork ALL=(ALL) NOPASSWD: ALL\n" + + +def test_the_rule_has_a_file_of_its_own_no_personal_rule_shares() -> None: + path = service_sudoers_rule_path("/etc/sudoers.d", "net.ork_1") + assert path == "/etc/sudoers.d/85-service-net_2eork__1" + assert path != sudoers_rule_path("/etc/sudoers.d", "net.ork_1") + + +def test_a_new_service_account_gets_the_key_and_sudo_without_a_password( + host: _ServiceHost, driver: _Driver +) -> None: + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["account", "sudo", "keys"] + assert host.accounts["netork"].password is None + assert host.files["/home/netork/.ssh/authorized_keys"] == NETORK_KEY + "\n" + assert host.is_passwordless("netork") + account = result["account"] + assert account is not None and account["admin"] + + +def test_every_command_root_runs_is_one_fixed_command(host: _ServiceHost, driver: _Driver) -> None: + driver.add_service_account("netork", NETORK_KEY) + + for command in _root_commands(host): + assert not any(s in command for s in SHELL_SYNTAX), command + + +def test_an_existing_account_keeps_its_keys_and_gets_the_rule( + host: _ServiceHost, driver: _Driver +) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + host.files["/home/netork/.ssh/authorized_keys"] = OWN_KEY + "\n" + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert host.files["/home/netork/.ssh/authorized_keys"] == f"{OWN_KEY}\n{NETORK_KEY}\n" + assert host.is_passwordless("netork") + + +def test_an_existing_system_account_may_become_netorks(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["netork"] = _Account(998, "/var/lib/netork", groups=["netork"]) + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + + +def test_a_locked_account_is_opened_again(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"], locked=True) + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert not host.accounts["netork"].locked + + +def test_the_account_netork_logs_in_with_already_only_gets_what_it_lacks( + host: _ServiceHost, driver: _Driver +) -> None: + host.login = "netork" + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + host.files["/home/netork/.ssh/authorized_keys"] = NETORK_KEY + "\n" + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert not any(c.startswith("openuser") for c in _root_commands(host)) + assert "already" in result["output"] + + +def test_root_is_never_netorks_account(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["toor"] = _Account(0, "/root", groups=["wheel"]) + + with pytest.raises(ValueError, match="root"): + driver.add_service_account("toor", NETORK_KEY) + assert not _root_commands(host) + + +@pytest.mark.parametrize("name", ["Netork", "net ork", "-n", ""]) +def test_a_name_no_host_takes_is_refused(driver: _Driver, name: str) -> None: + with pytest.raises(ValueError): + driver.add_service_account(name, NETORK_KEY) + + +def test_a_key_that_is_none_is_refused(driver: _Driver) -> None: + with pytest.raises(ValueError): + driver.add_service_account("netork", "not a key") + + +def test_a_rule_sudo_does_not_read_fails_the_step(host: _ServiceHost, driver: _Driver) -> None: + host.includes_dropins = False + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "does not read it" in sudo["output"] + + +def test_a_rule_with_a_password_is_not_enough(host: _ServiceHost, driver: _Driver) -> None: + host.other_rules.add("netork") + host.includes_dropins = False + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + + +def test_a_failed_create_stops_before_sudo_and_keys(host: _ServiceHost, driver: _Driver) -> None: + host.failing.add("mkuser") + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + assert [s["step"] for s in result["steps"]] == ["account"] + assert result["account"] is None + + +def test_the_staged_files_are_removed_again(host: _ServiceHost, driver: _Driver) -> None: + driver.add_service_account("netork", NETORK_KEY) + + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_a_personal_account_of_the_same_name_keeps_its_own_rule_apart( + host: _ServiceHost, driver: _Driver +) -> None: + driver.add_service_account("netork", NETORK_KEY) + + rules = [c for c in _root_commands(host) if c.startswith("install")] + assert [shlex.split(c)[-1] for c in rules] == [ + service_sudoers_rule_path(_Driver.SUDOERS_DIR, "netork") + ] From 3f59023f4674201ce69ffad55d5474a38054afb4 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 12:42:30 +0100 Subject: [PATCH 37/50] feat: a fixed uid for netOrk's own account (NetOrk/netork#1817) add_service_account(uid=) makes a new account with that uid once no other account has it, through the driver's _account_create_with_uid; an account that is there keeps its own, as a uid is never changed under files that carry it. The infrastructure's build standard wants the same uid on every host. --- CHANGELOG.md | 4 +- napalm_device_types/local_accounts.py | 54 ++++++++++++++++- tests/test_service_account.py | 83 +++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f1c27e..d6a8ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ recorded here; their history is the git log. password, and a rule of its own in `85-service-` lets it run everything as root without a password (`NOPASSWD: ALL`, checked with `visudo` and `sudo -l`). An account that is there is opened again and keeps its other keys; root is refused - (NetOrk/netork#1817). + (NetOrk/netork#1817). With `uid=`, a new account gets that uid once no other account + has it; one that is there keeps its own. A driver supplies + `_account_create_with_uid`. - `LocalAccountsMixin` (`napalm_device_types.local_accounts`): personal login accounts on a host. `get_local_accounts(names)` reads them back (groups, lock, sudo, key files), `add_local_account(name, public_key, password_hash=, admin=)` creates or diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py index 9cde5c3..fae3d6e 100644 --- a/napalm_device_types/local_accounts.py +++ b/napalm_device_types/local_accounts.py @@ -254,6 +254,14 @@ class LocalAccountsMixin: """The command that takes *name* out of *group*; *groups* are all of its own.""" raise NotImplementedError + def _account_create_with_uid(self, name: str, uid: int) -> list[list[str]]: + """The commands that create *name* as :meth:`_account_create` does, without a + password, and with the uid *uid*. + + Only netOrk's own account asks for one (``add_service_account(uid=)``). + """ + raise NotImplementedError + # -- reading --------------------------------------------------------------------- def get_local_accounts(self, names: Iterable[str]) -> list[LocalAccountDict]: @@ -383,7 +391,9 @@ class LocalAccountsMixin: ] return self._accounts_result(name, parts) - def add_service_account(self, name: str, public_key: str) -> LocalAccountChangeDict: + def add_service_account( + self, name: str, public_key: str, *, uid: int | None = None + ) -> LocalAccountChangeDict: """Make *name* netOrk's own account: it logs in with *public_key*, and gains root without a password. @@ -392,14 +402,20 @@ class LocalAccountsMixin: again with the account the driver already logs in with, it only adds what is missing. + A new account gets *uid* when one is given -- the same on every host -- + once no other account has it; an account that is there keeps its own, + as a uid is never changed under files that carry it. + Steps as in :meth:`add_local_account`: ``account``, then ``sudo`` and ``keys``. The new login itself is for the caller to try. - :raises ValueError: for a name or key no host takes, or root. + :raises ValueError: for a name or key no host takes, root, or a uid of 0 or less. :raises RuntimeError: when the login user cannot be told. """ check_account_name(name) check_public_key(public_key) + if uid is not None and uid <= 0: + raise ValueError(f"not a uid for netOrk's own account: {uid}") login = self._accounts_login() existing = self._accounts_passwd(name) if existing is not None and int(existing[2]) == 0: @@ -413,7 +429,7 @@ class LocalAccountsMixin: first: FixResultDict = _joined([step]) else: first = run_fix_steps( - [("account", lambda: self._accounts_open_or_create(name, existing, None))] + [("account", lambda: self._accounts_service_account(name, existing, uid))] ) parts = [first] if first["success"]: @@ -423,6 +439,38 @@ class LocalAccountsMixin: ] return self._accounts_result(name, parts) + def _accounts_service_account( + self, name: str, passwd: list[str] | None, uid: int | None + ) -> FixStepOutcomeDict: + """netOrk's own account opened or made, a new one with *uid* when given.""" + if uid is None: + return self._accounts_open_or_create(name, passwd, None) + if passwd is not None: + outcome = self._accounts_open_or_create(name, passwd, None) + if outcome["ok"] and int(passwd[2]) != uid: + outcome["output"] += f"; it keeps its uid {passwd[2]}, not {uid}" + return outcome + owner = self._accounts_passwd_of_uid(uid) + if owner is not None: + return {"ok": False, "output": f"uid {uid} is the account {owner}'s"} + try: + create = self._account_create_with_uid(name, uid) + except NotImplementedError: + return {"ok": False, "output": f"this driver cannot give an account the uid {uid}"} + failed = self._accounts_run(create) + if failed is not None: + return {"ok": False, "output": _said(failed)} + return {"ok": True, "output": f"account {name} created, uid {uid}"} + + def _accounts_passwd_of_uid(self, uid: int) -> str | None: + """The name of the account with *uid*, or None.""" + result = self.run_command(_command(["getent", "passwd", str(uid)]), timeout=30) + if result.exit_code == 2: + return None + if result.exit_code != 0: + raise RuntimeError(f"cannot ask who has uid {uid}: {_said(result)}") + return result.stdout.split(":", 1)[0].strip() or None + def _accounts_check_hash(self, password_hash: str) -> None: prefixes = PASSWORD_SCHEMES[self.ACCOUNT_PASSWORD_SCHEME] if not _CRYPT_RE.fullmatch(password_hash) or not password_hash.startswith(prefixes): diff --git a/tests/test_service_account.py b/tests/test_service_account.py index 0c53bb7..d9d4169 100644 --- a/tests/test_service_account.py +++ b/tests/test_service_account.py @@ -197,3 +197,86 @@ def test_a_personal_account_of_the_same_name_keeps_its_own_rule_apart( assert [shlex.split(c)[-1] for c in rules] == [ service_sudoers_rule_path(_Driver.SUDOERS_DIR, "netork") ] + + +# -- a fixed uid (infrastructure BAUSTANDARD A) ------------------------------------------ + + +class _UidDriver(_Driver): + def _account_create_with_uid(self, name: str, uid: int) -> list[list[str]]: + return [["mkuseruid", name, str(uid)]] + + +def _mkuseruid(host: _ServiceHost): + def handler(args: list[str], privileged: bool) -> CommandResult: + assert privileged + name, uid = args[0], int(args[1]) + host.accounts[name] = _Account(uid, f"/home/{name}", groups=[name]) + return CommandResult("", "", 0) + + return handler + + +def _getent_uid(host: _ServiceHost, original): + def handler(args: list[str], privileged: bool) -> CommandResult: + if args[1].isdigit(): + owner = next((n for n, a in host.accounts.items() if a.uid == int(args[1])), None) + if owner is None: + return CommandResult("", "", 2) + a = host.accounts[owner] + return CommandResult(f"{owner}:*:{a.uid}:{a.uid}::{a.home}:{a.shell}\n", "", 0) + return original(args, privileged) + + return handler + + +@pytest.fixture +def uid_driver(host: _ServiceHost) -> _UidDriver: + host._mkuseruid = _mkuseruid(host) # type: ignore[attr-defined] + host._getent = _getent_uid(host, host._getent) # type: ignore[method-assign] + return _UidDriver(host) + + +def test_a_new_account_gets_the_uid_asked_for(host: _ServiceHost, uid_driver: _UidDriver) -> None: + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert result["success"], result["output"] + assert host.accounts["netork"].uid == 900 + account = result["account"] + assert account is not None and account["uid"] == 900 + + +def test_a_uid_another_account_has_stops_before_anything_is_made( + host: _ServiceHost, uid_driver: _UidDriver +) -> None: + host.accounts["fwupd-refresh"] = _Account(900, "/run/fwupd") + + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert not result["success"] + assert "fwupd-refresh" in result["output"] + assert "netork" not in host.accounts + + +def test_an_existing_account_keeps_its_uid_and_says_so( + host: _ServiceHost, uid_driver: _UidDriver +) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert result["success"], result["output"] + assert host.accounts["netork"].uid == 1001 + assert "uid 1001" in result["output"] + + +def test_a_uid_root_has_is_refused(uid_driver: _UidDriver) -> None: + with pytest.raises(ValueError): + uid_driver.add_service_account("netork", NETORK_KEY, uid=0) + + +def test_a_driver_that_cannot_set_a_uid_says_so(host: _ServiceHost, driver: _Driver) -> None: + result = driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert not result["success"] + assert "uid" in result["output"] From 47751d12d623faae87f2af27e58c8532bd175142 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 13:36:38 +0100 Subject: [PATCH 38/50] fix: another service's account of the same name is left alone (NetOrk/netork#1817) On bw-01, netork is the satellite's own user (nologin, in docker). add_service_account opened it, gave it the passwordless sudo rule and netOrk's key, and only the login as it failed. An account that is there and cannot log in (nologin, false) is now refused before anything changes. --- CHANGELOG.md | 4 ++- napalm_device_types/local_accounts.py | 35 +++++++++++++++++++++------ tests/test_service_account.py | 19 +++++++++++++++ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6a8ea8..58e1ba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,9 @@ recorded here; their history is the git log. account that is there is opened again and keeps its other keys; root is refused (NetOrk/netork#1817). With `uid=`, a new account gets that uid once no other account has it; one that is there keeps its own. A driver supplies - `_account_create_with_uid`. + `_account_create_with_uid`. An account that is there and cannot log in (`nologin`, + `false`) is another service's, such as a netOrk satellite's `netork`, and is refused + before anything changes. - `LocalAccountsMixin` (`napalm_device_types.local_accounts`): personal login accounts on a host. `get_local_accounts(names)` reads them back (groups, lock, sudo, key files), `add_local_account(name, public_key, password_hash=, admin=)` creates or diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py index fae3d6e..d7f7b1e 100644 --- a/napalm_device_types/local_accounts.py +++ b/napalm_device_types/local_accounts.py @@ -85,6 +85,9 @@ _RUN_ALL_NOPASSWD_RE = re.compile( re.MULTILINE, ) +#: Login shells that refuse every login: an account with one belongs to a service. +_NO_LOGIN_SHELLS = frozenset({"nologin", "false"}) + #: Stages stdin as a file anyone may read -- the person reads the key file #: from it -- and prints its path. The content is a public key or a sudo rule. _STAGE = 't=$(mktemp) && chmod 644 "$t" && cat > "$t" && printf %s "$t"' @@ -409,17 +412,15 @@ class LocalAccountsMixin: Steps as in :meth:`add_local_account`: ``account``, then ``sudo`` and ``keys``. The new login itself is for the caller to try. - :raises ValueError: for a name or key no host takes, root, or a uid of 0 or less. + An account that is there and cannot log in (``nologin``, ``false``) is + another service's -- a satellite's ``netork``, say -- and is left alone. + + :raises ValueError: for a name or key no host takes, root, a uid of 0 or + less, or another service's account. :raises RuntimeError: when the login user cannot be told. """ - check_account_name(name) check_public_key(public_key) - if uid is not None and uid <= 0: - raise ValueError(f"not a uid for netOrk's own account: {uid}") - login = self._accounts_login() - existing = self._accounts_passwd(name) - if existing is not None and int(existing[2]) == 0: - raise ValueError(f"{name} is root") + login, existing = self._accounts_service_target(name, uid) if name == login: step: FixStepDict = { "step": "account", @@ -439,6 +440,24 @@ class LocalAccountsMixin: ] return self._accounts_result(name, parts) + def _accounts_service_target(self, name: str, uid: int | None) -> tuple[str, list[str] | None]: + """The login user and *name*'s passwd line, once *name* may become netOrk's. + + :raises ValueError: as :meth:`add_service_account` says. + """ + check_account_name(name) + if uid is not None and uid <= 0: + raise ValueError(f"not a uid for netOrk's own account: {uid}") + login = self._accounts_login() + existing = self._accounts_passwd(name) + if existing is not None and int(existing[2]) == 0: + raise ValueError(f"{name} is root") + if existing is not None and existing[6].rsplit("/", 1)[-1] in _NO_LOGIN_SHELLS: + raise ValueError( + f"{name} is there and cannot log in ({existing[6]}): another service's account" + ) + return login, existing + def _accounts_service_account( self, name: str, passwd: list[str] | None, uid: int | None ) -> FixStepOutcomeDict: diff --git a/tests/test_service_account.py b/tests/test_service_account.py index d9d4169..9bc270a 100644 --- a/tests/test_service_account.py +++ b/tests/test_service_account.py @@ -280,3 +280,22 @@ def test_a_driver_that_cannot_set_a_uid_says_so(host: _ServiceHost, driver: _Dri assert not result["success"] assert "uid" in result["output"] + + +# -- another service's account of the same name (bw-01: the satellite's) ------------------ + + +@pytest.mark.parametrize( + "shell", ["/usr/sbin/nologin", "/sbin/nologin", "/bin/false", "/usr/bin/false"] +) +def test_an_account_that_cannot_log_in_is_another_services_and_left_alone( + host: _ServiceHost, driver: _Driver, shell: str +) -> None: + host.accounts["netork"] = _Account( + 999, "/home/netork", shell=shell, groups=["netork", "docker"] + ) + + with pytest.raises(ValueError, match="cannot log in"): + driver.add_service_account("netork", NETORK_KEY) + assert not _root_commands(host) + assert service_sudoers_rule_path(_Driver.SUDOERS_DIR, "netork") not in host.files From 00aa5efef01765f5678d63a3ed8538d576b4a6cf Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 16:38:38 +0100 Subject: [PATCH 39/50] feat: get_clusters() and create_vm_from_cloud_init(node=) A hypervisor reports every cluster its connection can see: the cluster's name and quorum, its nodes (offline ones too, the local one marked) with CPU and memory, and the storages and networks with the nodes that see each. A standalone host is a cluster of one without a name; a manager reports each cluster and standalone host it runs. Declared only, so a capability check stays truthful. create_vm_from_cloud_init() takes the node to create the VM on; a driver that can only create on its own node refuses another. Refs NetOrk/netork#484, NetOrk/netork#1822 --- CHANGELOG.md | 9 +++ napalm_device_types/__init__.py | 8 +++ napalm_device_types/hypervisor.py | 33 +++++++++ napalm_device_types/models.py | 51 +++++++++++++ tests/test_clusters.py | 114 ++++++++++++++++++++++++++++++ 5 files changed, 215 insertions(+) create mode 100644 tests/test_clusters.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 58e1ba6..9ad65a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,15 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, + `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, + with its nodes (offline ones included, `local` on the node the connection is), + their CPU and memory, and the storages and networks with the nodes that see each. + A standalone host is a cluster of one with no name; a manager reports each cluster + and standalone host it runs. Declared only (NetOrk/netork#484, #1821). +- `create_vm_from_cloud_init(node=…)`: the cluster node to create the VM on. None + leaves it to the driver; a driver that can only create on its own node raises + ValueError for another (NetOrk/netork#1822). - `LocalAccountsMixin.add_service_account(name, public_key)`: netOrk's own account, created through the account the driver logs in with today. It takes a key and no password, and a rule of its own in `85-service-` lets it run everything as diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index a3d1e8f..f455983 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -135,6 +135,10 @@ from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver from napalm_device_types.models import ( POWER_ACTIONS, + ClusterDict, + ClusterNetworkDict, + ClusterNodeDict, + ClusterStorageDict, ConfigSyncDict, DhcpHaDict, DhcpHaPeerDict, @@ -235,6 +239,10 @@ from napalm_device_types.virtual_ips import ( from napalm_device_types.vm_stop_request import VmStopRequestMixin __all__ = [ + "ClusterDict", + "ClusterNetworkDict", + "ClusterNodeDict", + "ClusterStorageDict", "HypervisorHostDict", "DnsForwardDict", "DnsForwardMixin", diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 86c3d42..655a709 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( + ClusterDict, HypervisorHostDict, NetworkTargetDict, NICConfigDict, @@ -464,6 +465,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri storage: str | None = None, cpu_type: str | None = None, guest_os: str = "linux", + node: str | None = None, download_timeout: int = 300, timeout: int = 180, ) -> VMProvisionResultDict: @@ -519,6 +521,13 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri network-config itself (``provisioning.network_config``). A guest_os the driver does not list raises ValueError before anything is created. Default ``"linux"``. + node (string | None) - the cluster node to create the VM on, a + node ``name`` from ``get_clusters()``. None leaves it to the + driver: the node this connection talks to, or, on a manager of + several hosts, a host of the driver's choosing. A driver that + can only create on its own node raises ValueError for any other + name, before creating anything. ``storage`` and the NICs' + bridges must then be ones that node sees. download_timeout (int) - maximum seconds to wait for the image download (skipped entirely if already cached on the hypervisor). Default 300. timeout (int) - maximum seconds to wait for the remaining provisioning @@ -637,6 +646,30 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... + def get_clusters(self) -> list[ClusterDict]: + """ + List the clusters this connection can see, each with its nodes, its + storages and its networks. + + A cluster member (a Proxmox node) reports its one cluster, the whole of + it: every node, offline ones included with ``status`` ``"offline"``, and + ``local`` set on the node it is. A manager (vCenter) reports each + cluster it runs, and each standalone host it runs as a cluster of one. + A standalone host reports itself as a cluster of one, ``name`` None. + + The VMs ``get_vms()`` lists name their node in ``node``; that name is + the node's ``name`` here. A storage or network lists the nodes it is + available on, so a caller can tell where a VM may go. A value the + platform does not report is None, never a guess. + + Facts only: which node is a good place for a VM is the caller's to + decide. + + Returns: + List[ClusterDict] + """ + ... + def get_hypervisor_hosts(self) -> list[HypervisorHostDict]: """The hosts this hypervisor or its manager runs, and how each stands: connected, in maintenance, with configuration issues, under a central diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 112a57c..3819fa1 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1195,6 +1195,57 @@ class HypervisorHostDict(TypedDict): managed_by: NotRequired[str | None] # the address of a central manager, if any +class ClusterNodeDict(TypedDict): + """One host of a cluster, as its cluster sees it. A value the platform does not + report is None; it is never guessed.""" + + name: str # the node's name in its cluster, as get_vms() reports it in ``node`` + status: str # "online", "offline" or "unknown" + local: bool # the node this connection talks to; False on a manager (vCenter) + maintenance: bool | None # in maintenance mode + cpu_threads: int | None # logical CPUs + cpu_cores: int | None # physical cores + cpu_model: str | None + cpu_flags: list[str] | None # cpuinfo flags; None where the platform does not say + cpu_usage: float | None # 0.0–1.0 + memory_total: int | None # bytes + memory_used: int | None # bytes + + +class ClusterStorageDict(TypedDict): + """A storage of a cluster and the nodes that see it.""" + + name: str # usable as create_vm_from_cloud_init(storage=...) on each of ``nodes`` + type: str # backend type: "dir", "lvmthin", "zfspool", "nfs", "rbd", "vmfs", ... + shared: bool # the same storage on every node in ``nodes``, so a VM on it can move + content: list[str] # what it holds: "images", "iso", "snippets", ... + nodes: list[str] # the nodes it is available on + total: int | None # bytes; None where the platform does not report it + used: int | None # bytes + + +class ClusterNetworkDict(TypedDict): + """A network a VM's NIC may attach to, and the nodes that carry it.""" + + name: str # usable as NICConfigDict.bridge on each of ``nodes`` + kind: str # "bridge", "vnet" or "portgroup", as in NetworkTargetDict + vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of it + fixed_vlan_tag: NotRequired[int | None] # vnet/portgroup: the VLAN baked into it + nodes: list[str] + + +class ClusterDict(TypedDict): + """A cluster the connection can see. A standalone host is a cluster of one, + with no name.""" + + id: str | None # stable across connections where the platform has one; None otherwise + name: str | None # the cluster's name; None for a standalone host + quorate: bool | None # None where the platform has no quorum (standalone, vSphere) + nodes: list[ClusterNodeDict] + storages: list[ClusterStorageDict] + networks: list[ClusterNetworkDict] + + class FixStepOutcomeDict(TypedDict): """What one step of a fix routine reports (see ``napalm_device_types.fixes``).""" diff --git a/tests/test_clusters.py b/tests/test_clusters.py new file mode 100644 index 0000000..30e537c --- /dev/null +++ b/tests/test_clusters.py @@ -0,0 +1,114 @@ +"""A hypervisor says which cluster it belongs to, and what that cluster holds. + +A Proxmox node is one member of a cluster and can read the whole of it; a vCenter +manages several clusters and standalone hosts at once; a standalone host is a +cluster of one. One getter serves all three: ``get_clusters()`` lists every +cluster the connection can see, each with its nodes, its storages and its +networks, and which nodes see which. What a caller does with that (placement, +headroom, identity) is the caller's. + +A new VM can be placed on a named node (``create_vm_from_cloud_init(node=…)``), +so a manager that runs several hosts takes the caller's choice instead of making +its own. + +The declarations sit under ``TYPE_CHECKING`` (see test_role_contracts), so the +signatures are read from the source rather than from the class. +""" + +from __future__ import annotations + +import ast +import inspect +from typing import NotRequired, get_type_hints + +import napalm_device_types.hypervisor as hypervisor_module +from napalm_device_types import ( + ClusterDict, + ClusterNetworkDict, + ClusterNodeDict, + ClusterStorageDict, + HypervisorDriver, +) + + +def _declared(name: str) -> ast.FunctionDef: + tree = ast.parse(inspect.getsource(hypervisor_module)) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == name: + return node + raise AssertionError(f"HypervisorDriver does not declare {name}()") + + +class TestClusterListing: + def test_is_declared(self): + fn = _declared("get_clusters") + assert ast.unparse(fn.returns) == "list[ClusterDict]" + + def test_absent_until_a_driver_implements_it(self): + """netOrk probes capabilities with hasattr; a hypervisor that cannot + describe its cluster must not seem to.""" + assert not hasattr(HypervisorDriver, "get_clusters") + + +class TestClusterShape: + def test_cluster(self): + assert get_type_hints(ClusterDict) == { + "id": str | None, + "name": str | None, + "quorate": bool | None, + "nodes": list[ClusterNodeDict], + "storages": list[ClusterStorageDict], + "networks": list[ClusterNetworkDict], + } + + def test_node(self): + hints = get_type_hints(ClusterNodeDict) + assert hints == { + "name": str, + "status": str, + "local": bool, + "maintenance": bool | None, + "cpu_threads": int | None, + "cpu_cores": int | None, + "cpu_model": str | None, + "cpu_flags": list[str] | None, + "cpu_usage": float | None, + "memory_total": int | None, + "memory_used": int | None, + } + + def test_storage(self): + assert get_type_hints(ClusterStorageDict) == { + "name": str, + "type": str, + "shared": bool, + "content": list[str], + "nodes": list[str], + "total": int | None, + "used": int | None, + } + + def test_network(self): + hints = get_type_hints(ClusterNetworkDict, include_extras=True) + assert hints == { + "name": str, + "kind": str, + "vlan_aware": bool, + "fixed_vlan_tag": NotRequired[int | None], + "nodes": list[str], + } + assert ClusterNetworkDict.__required_keys__ == frozenset( + {"name", "kind", "vlan_aware", "nodes"} + ) + + +class TestCreateVmTakesANode: + def test_node_is_an_optional_keyword(self): + fn = _declared("create_vm_from_cloud_init") + kwonly = { + arg.arg: default + for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults, strict=True) + } + assert "node" in kwonly + default = kwonly["node"] + assert isinstance(default, ast.Constant) and default.value is None From 5b894913e1eaa6d0cf2d38081a8bb70c7c450175 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 17:08:10 +0100 Subject: [PATCH 40/50] refactor: the cluster shapes in clusters.py, a shorter provisioning docstring models.py is over the file size limit and may not grow, and the declaration of create_vm_from_cloud_init() may not get longer: the four cluster shapes move to napalm_device_types/clusters.py (exported from the package as before), and the docstring says the same in fewer lines. Refs NetOrk/netork#484 --- napalm_device_types/__init__.py | 10 +++--- napalm_device_types/clusters.py | 57 +++++++++++++++++++++++++++++++ napalm_device_types/hypervisor.py | 22 ++++-------- napalm_device_types/models.py | 51 --------------------------- 4 files changed, 70 insertions(+), 70 deletions(-) create mode 100644 napalm_device_types/clusters.py diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index f455983..743e0d5 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -87,6 +87,12 @@ from napalm_device_types.channel import ( open_stream_on_transport, run_on_transport, ) +from napalm_device_types.clusters import ( + ClusterDict, + ClusterNetworkDict, + ClusterNodeDict, + ClusterStorageDict, +) from napalm_device_types.config_lifecycle import ConfigLifecycleMixin from napalm_device_types.config_sync import ConfigSyncMixin from napalm_device_types.container_engine import ( @@ -135,10 +141,6 @@ from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver from napalm_device_types.models import ( POWER_ACTIONS, - ClusterDict, - ClusterNetworkDict, - ClusterNodeDict, - ClusterStorageDict, ConfigSyncDict, DhcpHaDict, DhcpHaPeerDict, diff --git a/napalm_device_types/clusters.py b/napalm_device_types/clusters.py new file mode 100644 index 0000000..c5e136e --- /dev/null +++ b/napalm_device_types/clusters.py @@ -0,0 +1,57 @@ +"""The shapes of a hypervisor's clusters (``HypervisorDriver.get_clusters()``). + +Apart from ``models.py``, which is over the file size limit. +""" + +from typing import NotRequired, TypedDict + + +class ClusterNodeDict(TypedDict): + """One host of a cluster, as its cluster sees it. A value the platform does not + report is None; it is never guessed.""" + + name: str # the node's name in its cluster, as get_vms() reports it in ``node`` + status: str # "online", "offline" or "unknown" + local: bool # the node this connection talks to; False on a manager (vCenter) + maintenance: bool | None # in maintenance mode + cpu_threads: int | None # logical CPUs + cpu_cores: int | None # physical cores + cpu_model: str | None + cpu_flags: list[str] | None # cpuinfo flags; None where the platform does not say + cpu_usage: float | None # 0.0–1.0 + memory_total: int | None # bytes + memory_used: int | None # bytes + + +class ClusterStorageDict(TypedDict): + """A storage of a cluster and the nodes that see it.""" + + name: str # usable as create_vm_from_cloud_init(storage=...) on each of ``nodes`` + type: str # backend type: "dir", "lvmthin", "zfspool", "nfs", "rbd", "vmfs", ... + shared: bool # the same storage on every node in ``nodes``, so a VM on it can move + content: list[str] # what it holds: "images", "iso", "snippets", ... + nodes: list[str] # the nodes it is available on + total: int | None # bytes; None where the platform does not report it + used: int | None # bytes + + +class ClusterNetworkDict(TypedDict): + """A network a VM's NIC may attach to, and the nodes that carry it.""" + + name: str # usable as NICConfigDict.bridge on each of ``nodes`` + kind: str # "bridge", "vnet" or "portgroup", as in NetworkTargetDict + vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of it + fixed_vlan_tag: NotRequired[int | None] # vnet/portgroup: the VLAN baked into it + nodes: list[str] + + +class ClusterDict(TypedDict): + """A cluster the connection can see. A standalone host is a cluster of one, + with no name.""" + + id: str | None # stable across connections where the platform has one; None otherwise + name: str | None # the cluster's name; None for a standalone host + quorate: bool | None # None where the platform has no quorum (standalone, vSphere) + nodes: list[ClusterNodeDict] + storages: list[ClusterStorageDict] + networks: list[ClusterNetworkDict] diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 655a709..3618c22 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -13,9 +13,9 @@ Usage:: from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver +from napalm_device_types.clusters import ClusterDict from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( - ClusterDict, HypervisorHostDict, NetworkTargetDict, NICConfigDict, @@ -478,9 +478,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri configuration via a storage snippet or similar mechanism. The resulting VM is left in a running state. - Implementations should cache downloaded images by URL/filename on the - hypervisor so repeated provisioning from the same image does not - re-download it every time. + Implementations cache downloaded images on the hypervisor by URL/filename. Args: name (string) - new VM display name @@ -499,10 +497,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri image_checksum (string | None) - expected checksum of the downloaded image (e.g. "sha256:"). If given, verified after download; mismatch raises RuntimeError. If None, no verification is performed. - ssh_public_keys (list[str] | None) - SSH public keys to inject into guest. - If None or empty, no SSH key injection is performed. - disk_resize_gb (int | None) - resize root disk to this size in GB. - If None, disk remains the downloaded image's native size. Default None. + ssh_public_keys (list[str] | None) - SSH public keys to inject (None: none). + disk_resize_gb (int | None) - resize root disk to this size in GB (None: keep). storage (string | None) - name of the storage pool to place the root disk on (a name returned by ``get_image_storages()``). If None, the driver auto-detects the first enabled, node-available storage @@ -521,13 +517,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri network-config itself (``provisioning.network_config``). A guest_os the driver does not list raises ValueError before anything is created. Default ``"linux"``. - node (string | None) - the cluster node to create the VM on, a - node ``name`` from ``get_clusters()``. None leaves it to the - driver: the node this connection talks to, or, on a manager of - several hosts, a host of the driver's choosing. A driver that - can only create on its own node raises ValueError for any other - name, before creating anything. ``storage`` and the NICs' - bridges must then be ones that node sees. + node (string | None) - a node ``name`` from ``get_clusters()`` to create + the VM on; None leaves it to the driver. One that only creates on its + own node raises ValueError for another, before creating anything. download_timeout (int) - maximum seconds to wait for the image download (skipped entirely if already cached on the hypervisor). Default 300. timeout (int) - maximum seconds to wait for the remaining provisioning diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 3819fa1..112a57c 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1195,57 +1195,6 @@ class HypervisorHostDict(TypedDict): managed_by: NotRequired[str | None] # the address of a central manager, if any -class ClusterNodeDict(TypedDict): - """One host of a cluster, as its cluster sees it. A value the platform does not - report is None; it is never guessed.""" - - name: str # the node's name in its cluster, as get_vms() reports it in ``node`` - status: str # "online", "offline" or "unknown" - local: bool # the node this connection talks to; False on a manager (vCenter) - maintenance: bool | None # in maintenance mode - cpu_threads: int | None # logical CPUs - cpu_cores: int | None # physical cores - cpu_model: str | None - cpu_flags: list[str] | None # cpuinfo flags; None where the platform does not say - cpu_usage: float | None # 0.0–1.0 - memory_total: int | None # bytes - memory_used: int | None # bytes - - -class ClusterStorageDict(TypedDict): - """A storage of a cluster and the nodes that see it.""" - - name: str # usable as create_vm_from_cloud_init(storage=...) on each of ``nodes`` - type: str # backend type: "dir", "lvmthin", "zfspool", "nfs", "rbd", "vmfs", ... - shared: bool # the same storage on every node in ``nodes``, so a VM on it can move - content: list[str] # what it holds: "images", "iso", "snippets", ... - nodes: list[str] # the nodes it is available on - total: int | None # bytes; None where the platform does not report it - used: int | None # bytes - - -class ClusterNetworkDict(TypedDict): - """A network a VM's NIC may attach to, and the nodes that carry it.""" - - name: str # usable as NICConfigDict.bridge on each of ``nodes`` - kind: str # "bridge", "vnet" or "portgroup", as in NetworkTargetDict - vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of it - fixed_vlan_tag: NotRequired[int | None] # vnet/portgroup: the VLAN baked into it - nodes: list[str] - - -class ClusterDict(TypedDict): - """A cluster the connection can see. A standalone host is a cluster of one, - with no name.""" - - id: str | None # stable across connections where the platform has one; None otherwise - name: str | None # the cluster's name; None for a standalone host - quorate: bool | None # None where the platform has no quorum (standalone, vSphere) - nodes: list[ClusterNodeDict] - storages: list[ClusterStorageDict] - networks: list[ClusterNetworkDict] - - class FixStepOutcomeDict(TypedDict): """What one step of a fix routine reports (see ``napalm_device_types.fixes``).""" From 4150570a9459369dbade6f50dd83af1296ea55e6 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:09:01 +0100 Subject: [PATCH 41/50] feat: VMProvisionIncomplete names the VM a failed creation left behind create_vm_from_cloud_init raised a plain error once the VM existed, so the caller had no id to remove it by. The new RuntimeError carries vmid and node, with the failure as its cause; a failure before the VM exists raises as before. Refs NAPALM/napalm-proxmox#28 --- CHANGELOG.md | 4 ++ README.md | 5 ++- napalm_device_types/__init__.py | 2 + napalm_device_types/hypervisor.py | 3 ++ napalm_device_types/provisioning.py | 29 ++++++++++++++- tests/test_vm_provision_incomplete.py | 53 +++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 tests/test_vm_provision_incomplete.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad65a9..55a31b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `VMProvisionIncomplete` (`napalm_device_types.provisioning`): what + `create_vm_from_cloud_init` raises when it fails after the VM was created. A + `RuntimeError` with the failure as its cause and the half-built VM's `vmid` and + `node`, so the caller can remove it with `destroy_vm` (NAPALM/napalm-proxmox#28). - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/README.md b/README.md index 8a4c22e..bfde346 100644 --- a/README.md +++ b/README.md @@ -547,7 +547,10 @@ any other. A QEMU-based driver takes `provisioning.QEMU_GUEST_AGENTS` (Linux, FreeBSD, OpenBSD). `provisioning.network_config()` writes the network-config v2 every cloud-init flavour reads (FreeBSD's nuageinit reads no other), and `provisioning.split_compression()` says how a packed image is -unpacked; both are the same for every hypervisor. +unpacked; both are the same for every hypervisor. A creation that fails once +the VM exists raises `VMProvisionIncomplete`, a `RuntimeError` that names the +half-built VM (`vmid`, `node`) for the caller to `destroy_vm`, with the failure +as its cause. ### OS / Linux diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 743e0d5..83c4c5a 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -178,6 +178,7 @@ from napalm_device_types.package_updates import ( from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.phone import PhoneDriver from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping +from napalm_device_types.provisioning import VMProvisionIncomplete from napalm_device_types.residential_gateway import ResidentialGatewayDriver from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of from napalm_device_types.service_clusters import ( @@ -353,6 +354,7 @@ __all__ = [ "VIRTUAL_IPS_COMMAND", "VirtualIpsMixin", "VmStopRequestMixin", + "VMProvisionIncomplete", "SERVICE_CLUSTERS_COMMAND", "ServiceClustersMixin", "parse_service_clusters", diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 3618c22..d6e3f17 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -532,6 +532,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri Raises: RuntimeError - if provisioning fails (download failure, checksum mismatch, storage unavailable, invalid config, timeout, etc.) + VMProvisionIncomplete - a RuntimeError for a failure after the VM + was created: it carries that VM's ``vmid`` and ``node``, so the + caller can ``destroy_vm`` it, and the failure as its cause. """ ... diff --git a/napalm_device_types/provisioning.py b/napalm_device_types/provisioning.py index 8879934..b479614 100644 --- a/napalm_device_types/provisioning.py +++ b/napalm_device_types/provisioning.py @@ -3,8 +3,9 @@ What provisioning a VM from a cloud image needs to know about the guest. The same for every hypervisor: which agent a guest runs so a QEMU-based hypervisor can read its IP, the network-config cloud-init and FreeBSD's -nuageinit both read, and how a packed cloud image is unpacked. How a -hypervisor attaches, boots and talks to the VM stays in its driver. +nuageinit both read, how a packed cloud image is unpacked, and how a failure +names the VM it left behind. How a hypervisor attaches, boots and talks to +the VM stays in its driver. """ from __future__ import annotations @@ -37,6 +38,30 @@ _DECOMPRESSORS = { } +class VMProvisionIncomplete(RuntimeError): + """``create_vm_from_cloud_init`` failed after the VM was created. + + The VM is still on the hypervisor, half built: ``vmid`` and ``node`` name it + as ``VMProvisionResultDict`` would have, so the caller can remove it with + ``destroy_vm``. The failure itself is the ``__cause__`` (``raise ... from``), + and the message is its own. A failure before the VM exists raises as before. + """ + + def __init__(self, message: str, *, vmid: str, node: str) -> None: + super().__init__(message) + self.vmid = vmid + self.node = node + + def __reduce__(self) -> tuple[Any, ...]: + # The keyword-only arguments are not in ``args``, which is all that + # pickle passes back by default. + return (_vm_provision_incomplete, (str(self), self.vmid, self.node)) + + +def _vm_provision_incomplete(message: str, vmid: str, node: str) -> VMProvisionIncomplete: + return VMProvisionIncomplete(message, vmid=vmid, node=node) + + def network_config(nics: list[tuple[str, bool]]) -> dict[str, Any] | None: """Network-config v2: DHCP on every ``(mac, dhcp)`` NIC that asks for it. diff --git a/tests/test_vm_provision_incomplete.py b/tests/test_vm_provision_incomplete.py new file mode 100644 index 0000000..ddaef8d --- /dev/null +++ b/tests/test_vm_provision_incomplete.py @@ -0,0 +1,53 @@ +"""A provisioning that failed after the hypervisor gave the VM its id says +which VM it left behind, so the caller can remove it (NAPALM/napalm-proxmox#28).""" + +from __future__ import annotations + +import pickle + +import pytest + +from napalm_device_types import VMProvisionIncomplete +from napalm_device_types.provisioning import VMProvisionIncomplete as from_module + + +def _raise_from(cause: Exception) -> VMProvisionIncomplete: + try: + try: + raise cause + except Exception as exc: + raise VMProvisionIncomplete(str(exc), vmid="101", node="pve1") from exc + except VMProvisionIncomplete as caught: + return caught + + +def test_it_is_exported_from_the_package_root(): + assert VMProvisionIncomplete is from_module + + +def test_it_names_the_vm_left_behind(): + caught = _raise_from(RuntimeError("disk import failed")) + assert (caught.vmid, caught.node) == ("101", "pve1") + + +def test_it_keeps_the_failure_and_its_words(): + cause = RuntimeError("disk import failed") + caught = _raise_from(cause) + assert caught.__cause__ is cause + assert str(caught) == "disk import failed" + + +def test_a_caller_catching_runtime_error_still_catches_it(): + # The contract has create_vm_from_cloud_init raise RuntimeError on failure. + with pytest.raises(RuntimeError): + raise VMProvisionIncomplete("start failed", vmid="7", node="n") + + +def test_vmid_and_node_are_keyword_only(): + with pytest.raises(TypeError): + VMProvisionIncomplete("failed", "101", "pve1") # type: ignore[misc] + + +def test_it_survives_pickling_as_a_task_result_would(): + caught = pickle.loads(pickle.dumps(VMProvisionIncomplete("failed", vmid="101", node="pve1"))) + assert (str(caught), caught.vmid, caught.node) == ("failed", "101", "pve1") From 9f6ffa8a25322455dfcbe3fc5dd5c3d33f05955e Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:11:19 +0100 Subject: [PATCH 42/50] feat(snmp): let a health profile mark a value unknown SnmpValueSpec gains `unknown`. A driver whose device's agent answers a value wrongly says so in its SNMP_HEALTH_PROFILE, and the application reports no value for it instead of the standard reading or a default. OpenBSD's net-snmp reports the CPU as never idle, so a reading of ssCpuIdle shows every OpenBSD host at 100 % (NAPALM/napalm-bsd#7). --- CHANGELOG.md | 4 ++++ README.md | 3 +++ napalm_device_types/models.py | 9 +++++++-- tests/test_snmp_health_profile.py | 12 +++++++++++- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad65a9..a946659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `SnmpValueSpec.unknown`: a driver whose device's agent answers a value wrongly says + so in its `SNMP_HEALTH_PROFILE` (`"cpu": {"unknown": True}`), and the application + reports no value for it -- not the standard reading, not 0. OpenBSD's net-snmp, + which reports the CPU as never idle, is the case (NAPALM/napalm-bsd#7). - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/README.md b/README.md index 8a4c22e..8d4f788 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,9 @@ class MySwitchDriver(SwitchDriver): A value is `oids` (the first that answers wins) or a `table` folded by `first`, `average` or `max`, with an optional `pattern` for an agent that answers with text. What the profile does not name, the application reads from the standard UCD-MIB and IF-MIB. +A value the agent gets wrong is `{"unknown": True}`: the application then reports no value +for it, rather than the standard reading or a default (OpenBSD's net-snmp reports its CPU +as never idle). `KernelFactsMixin` (`get_kernel_facts`) is the exception that is mixed in by a driver rather than by a role base: what a Linux kernel has built and loaded is read the same way diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 112a57c..86828ee 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -1237,12 +1237,17 @@ class SnmpValueSpec(TypedDict, total=False): ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a regular expression whose first group is the number, for an agent that answers with text. + + ``unknown`` says the agent answers this value wrongly, or not at all, where it + should: the application reports no value -- neither the standard reading nor a + default. It stands alone, without ``oids`` or ``table``. """ oids: list[str] table: str fold: Literal["first", "average", "max"] pattern: str + unknown: bool class SnmpMemorySpec(TypedDict, total=False): @@ -1267,8 +1272,8 @@ class SnmpHealthProfile(TypedDict, total=False): A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer the standard UCD-MIB and IF-MIB; the application polls and computes. What a - profile does not name is read the standard way. Uptime is in TimeTicks, CPU in - percent busy. + profile does not name is read the standard way, and what it marks ``unknown`` is + not read at all. Uptime is in TimeTicks, CPU in percent busy. """ uptime: SnmpValueSpec diff --git a/tests/test_snmp_health_profile.py b/tests/test_snmp_health_profile.py index c5b306b..c0f8054 100644 --- a/tests/test_snmp_health_profile.py +++ b/tests/test_snmp_health_profile.py @@ -22,7 +22,7 @@ from napalm_device_types import ( def test_a_profile_says_where_each_value_lives(): assert set(get_type_hints(SnmpHealthProfile)) == {"uptime", "cpu", "memory", "interfaces"} - assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern"} + assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern", "unknown"} assert set(get_type_hints(SnmpMemorySpec)) == {"total", "used", "free", "percent", "unit"} assert set(get_type_hints(SnmpInterfacesSpec)) == {"skip", "tx_errors_are_drops"} @@ -42,6 +42,16 @@ def test_a_driver_declares_its_profile_as_a_class_attribute(): assert Driver.SNMP_HEALTH_PROFILE["cpu"]["oids"][0] == "1.3.6.1.4.1.9.1" +def test_a_value_the_agent_gets_wrong_is_declared_unknown(): + """Neither the standard reading nor a default: the application reports none.""" + assert get_type_hints(SnmpValueSpec)["unknown"] is bool + + class Driver: + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = {"cpu": {"unknown": True}} + + assert Driver.SNMP_HEALTH_PROFILE["cpu"].get("unknown") is True + + def test_the_collector_in_the_base_package_warns_that_it_is_deprecated(): async def get(oid): return None From bc17862a4452da8537b059879a16a3df1a70fa8b Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:11:21 +0100 Subject: [PATCH 43/50] feat(fixes): configure_lldp_agent(interfaces) template LLDP getters used to install, start and reconfigure lldpd on their own, on every poll. Getters only read now, so starting the agent needs an action the application calls. The template installs the agent through ensure_package, has the driver write the interfaces it runs on, then enables and restarts it through manage_service. The caller names the interfaces: an agent on every interface also advertises on a WAN port. Refs NAPALM/napalm-openwrt#8 --- CHANGELOG.md | 7 +++ README.md | 5 +- napalm_device_types/__init__.py | 2 + napalm_device_types/fixes.py | 63 ++++++++++++++++++++++++- tests/test_fix_templates.py | 81 +++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad65a9..d1d64b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `LldpAgentMixin.configure_lldp_agent(interfaces)` (`napalm_device_types.fixes`): run + an LLDP agent on the interfaces the caller names. The template installs the agent + (`ensure_package(LLDP_AGENT_PACKAGE)`), has the driver write the interface list + (`_write_lldp_agent_config`), then enables and restarts the service + (`manage_service(LLDP_AGENT_SERVICE, …)`); both names default to `lldpd`. No + interface is a ValueError before anything changes. It is the action for what LLDP + getters used to do on their own (NAPALM/napalm-openwrt#8). - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/README.md b/README.md index 8a4c22e..54a6d9a 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,9 @@ VM's clean shutdown without waiting for it — a hypervisor driver mixes it in w **Fixes are templates** (`napalm_device_types.fixes`, since 4.13). A change the application wants made on a device runs as fixed steps the driver implements, with values the caller supplies: `SnmpAgentMixin.configure_snmp_agent(settings)` (install, configure, -firewall, start), `EnsurePackageMixin.ensure_package(name)`, -`PackageProxyMixin.configure_package_proxy(url)` and +firewall, start), `LldpAgentMixin.configure_lldp_agent(interfaces)` (install, write the +interfaces, enable and restart; the driver writes only the interfaces), +`EnsurePackageMixin.ensure_package(name)`, `PackageProxyMixin.configure_package_proxy(url)` and `ContainerEngineMixin.grant_container_engine_access(user)`. A driver whose device takes the change in one call overrides the template method. A driver mixes in the fixes its device can take; none is on a role base. They replace `run_device_action`, which is deprecated. diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 743e0d5..3d114dd 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -107,6 +107,7 @@ from napalm_device_types.firewall_rules import FirewallRuleMixin from napalm_device_types.firmware import FirmwareOfferMixin from napalm_device_types.fixes import ( EnsurePackageMixin, + LldpAgentMixin, PackageProxyMixin, SnmpAgentMixin, run_fix_steps, @@ -260,6 +261,7 @@ __all__ = [ "FixResultDict", "FixStepDict", "FixStepOutcomeDict", + "LldpAgentMixin", "PackageProxyMixin", "APT_PROXY_COMMAND", "AptProxyMixin", diff --git a/napalm_device_types/fixes.py b/napalm_device_types/fixes.py index a3335d6..db1dc2f 100644 --- a/napalm_device_types/fixes.py +++ b/napalm_device_types/fixes.py @@ -22,7 +22,7 @@ They replace ``run_device_action`` (docs/STRUCTURE.md): from __future__ import annotations from collections.abc import Callable, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from napalm_device_types.models import ( FixResultDict, @@ -133,6 +133,66 @@ class EnsurePackageMixin: def install_package(self, name: str, version: str = "") -> None: ... +class LldpAgentMixin: + """Run an LLDP agent on the device, on the interfaces the caller names. + + The steps: install the agent, write the interfaces it runs on, enable it at boot + and restart it so it reads them. Installing and starting are generic, over + ``ensure_package`` and ``manage_service`` with ``LLDP_AGENT_PACKAGE`` and + ``LLDP_AGENT_SERVICE``; the driver writes the configuration. Which interfaces is + the caller's choice -- an agent on every interface also talks to the uplink of a + WAN port. Whether neighbours appear is the caller's to read with + ``get_lldp_neighbors``, after the agent's first advertisement interval. + """ + + #: The package that brings the agent, as ``ensure_package`` takes it. + LLDP_AGENT_PACKAGE: ClassVar[str] = "lldpd" + #: The agent's service, as ``manage_service`` takes it. + LLDP_AGENT_SERVICE: ClassVar[str] = "lldpd" + + def configure_lldp_agent(self, interfaces: Sequence[str]) -> FixResultDict: + """Install, configure for *interfaces*, enable and restart the agent. + + :raises ValueError: for no interface, before anything changes. + """ + names = [str(name) for name in interfaces] + if not names: + raise ValueError("name at least one interface for the LLDP agent") + return run_fix_steps( + [ + ("install", self._install_lldp_agent), + ("configure", lambda: self._write_lldp_agent_config(names)), + ("start", self._start_lldp_agent), + ] + ) + + def _install_lldp_agent(self) -> FixStepOutcomeDict: + result = self.ensure_package(self.LLDP_AGENT_PACKAGE) + return {"ok": bool(result["success"]), "output": str(result.get("output", ""))} + + def _start_lldp_agent(self) -> FixStepOutcomeDict: + service = self.LLDP_AGENT_SERVICE + for action in ("enable", "restart"): + result = self.manage_service(service, action) + if not result.get("success"): + output = str(result.get("output", "")).strip() + return {"ok": False, "output": f"{action} {service}: {output}".rstrip(": ")} + return {"ok": True, "output": f"{service} enabled and restarted"} + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def ensure_package(self, name: str) -> FixResultDict: ... + + def manage_service(self, name: str, action: str) -> dict[str, Any]: ... + + # The step a driver implements: + + def _write_lldp_agent_config(self, interfaces: list[str]) -> FixStepOutcomeDict: + """Have the agent send and listen on exactly *interfaces* (names as the + device's interface getters report them), replacing the list it had.""" + ... + + class PackageProxyMixin: """Point the device's package manager at a caching proxy.""" @@ -152,6 +212,7 @@ class PackageProxyMixin: __all__ = [ "EnsurePackageMixin", "FixStep", + "LldpAgentMixin", "PackageProxyMixin", "SnmpAgentMixin", "run_fix_steps", diff --git a/tests/test_fix_templates.py b/tests/test_fix_templates.py index e6b866f..6c1adfc 100644 --- a/tests/test_fix_templates.py +++ b/tests/test_fix_templates.py @@ -14,6 +14,7 @@ from napalm_device_types import ( EnsurePackageMixin, FirewallDriver, HypervisorDriver, + LldpAgentMixin, ManagementControllerDriver, MediaDriver, OSDriver, @@ -169,11 +170,91 @@ class TestEnsurePackage: assert "no candidate" in result["output"] +class _Lldp(LldpAgentMixin): + """The generic steps over ensure_package and manage_service; the driver's configure.""" + + def __init__(self, refuse: str = ""): + self.calls: list = [] + self.refuse = refuse + + def ensure_package(self, name): + self.calls.append(("ensure_package", name)) + ok = self.refuse != "install" + return {"success": ok, "steps": [], "output": "installed" if ok else "E: no candidate"} + + def _write_lldp_agent_config(self, interfaces): + self.calls.append(("configure", interfaces)) + return {"ok": self.refuse != "configure", "output": "interfaces set"} + + def manage_service(self, name, action): + self.calls.append((action, name)) + ok = self.refuse != action + return {"success": ok, "output": "" if ok else f"cannot {action}"} + + +class TestLldpAgent: + def test_install_configure_enable_and_restart_on_the_caller_s_interfaces(self): + agent = _Lldp() + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is True + assert agent.calls == [ + ("ensure_package", "lldpd"), + ("configure", ["br-lan"]), + ("enable", "lldpd"), + ("restart", "lldpd"), + ] + assert [s["step"] for s in result["steps"]] == ["install", "configure", "start"] + + def test_a_failed_install_configures_nothing(self): + agent = _Lldp(refuse="install") + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert agent.calls == [("ensure_package", "lldpd")] + assert "no candidate" in result["output"] + + def test_an_agent_that_will_not_start_at_boot_is_not_restarted(self): + agent = _Lldp(refuse="enable") + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert ("restart", "lldpd") not in agent.calls + assert "cannot enable" in result["output"] + + def test_a_failed_restart_fails_the_fix(self): + result = _Lldp(refuse="restart").configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert result["steps"][-1]["step"] == "start" + + def test_the_package_and_service_names_are_the_driver_s(self): + agent = _Lldp() + agent.LLDP_AGENT_PACKAGE = "lldpd-full" + agent.LLDP_AGENT_SERVICE = "lldpd.service" + + agent.configure_lldp_agent(["eth0"]) + + assert ("ensure_package", "lldpd-full") in agent.calls + assert ("restart", "lldpd.service") in agent.calls + + def test_no_interface_is_refused_before_anything_changes(self): + agent = _Lldp() + + with pytest.raises(ValueError): + agent.configure_lldp_agent([]) + assert agent.calls == [] + + @pytest.mark.parametrize("base", ROLE_BASES, ids=lambda b: b.__name__) @pytest.mark.parametrize( "method", [ "configure_snmp_agent", + "configure_lldp_agent", "ensure_package", "configure_package_proxy", "grant_container_engine_access", From 46c1c9dc4e963d948e0ad04a2f117563046f8044 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:13:20 +0100 Subject: [PATCH 44/50] feat: TLS certificates by trust on first use Devices ship self-signed certificates, so a driver either fails the CA check on every one of them or turns verification off and sends the login to whatever answers on the address. tls_certificates pins the certificate by its SHA-256 fingerprint instead, shaped like ssh_host_keys: the caller passes the fingerprint to expect by port (optional_args["tls_fingerprints"]), the driver refuses any other certificate before a request carries credentials, and every certificate seen goes to optional_args["tls_certificate_observer"]. Without a fingerprint the first certificate is accepted. CertificateCheck checks the certificate in a handshake of its own, then pins a requests session to it (urllib3's assert_fingerprint, direct and through a proxy), so no other host can step in between the check and the login. Refs NAPALM/napalm-hpe-aruba-procurve#17 --- CHANGELOG.md | 14 ++ README.md | 20 ++ napalm_device_types/tls_certificates.py | 285 ++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_tls_certificates.py | 285 ++++++++++++++++++++++++ 5 files changed, 605 insertions(+) create mode 100644 napalm_device_types/tls_certificates.py create mode 100644 tests/test_tls_certificates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad65a9..b12487a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,22 @@ recorded here; their history is the git log. on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). - The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python 3.12 venv no longer has. +- `requests` is a declared dependency (floor 2.25.0), for `tls_certificates`; it came + with napalm before. ### Added +- `napalm_device_types.tls_certificates`: TLS certificates by trust on first use, the + counterpart of `ssh_host_keys` for a driver's HTTPS API. A driver takes the + fingerprint to expect from `optional_args["tls_fingerprints"]` (by port, the SHA-256 + of the DER certificate, `SHA256:AB:CD:…`) and refuses any other certificate with + `TlsCertificateMismatch` (a `ConnectionException` naming both fingerprints) before a + password is sent; no CA is consulted, so a self-signed certificate verifies. Without + a fingerprint the first certificate is accepted. Every certificate seen goes to + `optional_args["tls_certificate_observer"]`. `CertificateCheck.verify_server()` + checks the certificate in a handshake of its own, `pin(session)` makes every later + connection of a `requests` session present the same one, `refusing()` turns a refusal + there into the mismatch, `observe()` reports it once the login stands + (NAPALM/napalm-hpe-aruba-procurve#17). - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/README.md b/README.md index 8a4c22e..ef39554 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,26 @@ caller's; `ssh_host_keys.HostKeyCheck` is how every SSH driver verifies against A caller that passes neither key gets first use on every connection, as before; a driver that ignores both still connects. Old and new versions on either side keep working. +### TLS certificates: trust on first use + +The same for a driver's HTTPS API. Devices ship self-signed certificates, so a CA check +fails on all of them and turning verification off trusts whatever answers; +`tls_certificates.CertificateCheck` pins the certificate by its fingerprint instead +(NAPALM/napalm-hpe-aruba-procurve#17). No CA is consulted. + +- **In:** `optional_args["tls_fingerprints"]` = `{"443": "SHA256:AB:CD:…"}`, by port: the + SHA-256 of the DER certificate as `openssl x509 -fingerprint -sha256` prints it (prefix, + colons and case optional). With one, any other certificate is refused *before* a request + carries credentials and the connection raises `TlsCertificateMismatch` (a + `ConnectionException` naming both fingerprints). Without one, the first certificate is + accepted. A driver does not fall back to another transport on a mismatch. +- **Out:** `optional_args["tls_certificate_observer"]`, a callable, gets one dict per + connection: `{"host", "port", "seen": {"fingerprint"}, "expected", "accepted"}`. +- **requests:** `check = CertificateCheck(optional_args, host, port)`; + `check.verify_server(timeout)` before the login, `check.pin(session)` so every later + connection of the session must present that certificate, the login inside + `with check.refusing():`, then `check.observe()`. + ## Design principle: generic vs. device-specific logic When adding behavior to a device-type base class, split it along one line: **would diff --git a/napalm_device_types/tls_certificates.py b/napalm_device_types/tls_certificates.py new file mode 100644 index 0000000..d6e315e --- /dev/null +++ b/napalm_device_types/tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +Network devices ship self-signed certificates no CA vouches for. A driver that +then turns verification off logs in with the device's credentials at whatever +answers on its address (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +The same answer as for SSH host keys (:mod:`napalm_device_types.ssh_host_keys`): +the layer above keeps the store of the certificates it trusts, a driver verifies. + +* It receives the fingerprints to expect in ``optional_args["tls_fingerprints"]`` + (:data:`KNOWN_FINGERPRINTS_ARG`), by the port they were seen on: + ``{"443": "SHA256:AB:CD:…"}``, the SHA-256 of the DER certificate as + ``openssl x509 -fingerprint -sha256`` prints it (the ``SHA256:`` prefix, + colons and case are optional, :func:`normalise_fingerprint`). With a + fingerprint, any other certificate is refused before a password leaves the + caller; no CA is consulted. Without one, the first certificate is accepted: + trust on first use. +* It reports every certificate it sees through + ``optional_args["tls_certificate_observer"]`` (:data:`OBSERVER_ARG`), a + callable that receives one dict per connection:: + + {"host": str, "port": int, + "seen": {"fingerprint"}, + "expected": {"fingerprint"} | None, + "accepted": bool} + + ``accepted`` is False only for a refused certificate; the connection then + raises :class:`TlsCertificateMismatch`. A failing observer is logged and + ignored. + +A caller that passes neither keeps working as before (first use, every time), +and a driver that ignores both still connects for a caller that passes them. + +:class:`CertificateCheck` is the one implementation. :meth:`~CertificateCheck.verify_server` +takes the certificate from a handshake of its own, before any request; +:meth:`~CertificateCheck.pin` makes every later connection of a ``requests`` +session present that same certificate (urllib3's ``assert_fingerprint``), so +another host cannot step in between the check and the login; +:meth:`~CertificateCheck.refusing` turns such a refusal into +:class:`TlsCertificateMismatch`; :meth:`~CertificateCheck.observe` reports the +certificate once the login stands. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import socket +import ssl +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any + +from napalm.base.exceptions import ConnectionException + +logger = logging.getLogger(__name__) + +#: The ``optional_args`` key that carries the fingerprints to expect, by port. +KNOWN_FINGERPRINTS_ARG = "tls_fingerprints" +#: The ``optional_args`` key that carries the callable every seen certificate goes to. +OBSERVER_ARG = "tls_certificate_observer" + +_PREFIX = "SHA256:" +_SHA256_HEX = re.compile(r"^[0-9A-F]{64}$") +#: What urllib3 says when a connection presents another certificate than the pinned one +#: (``got "…"``; 1.26 writes the digest as ``b'…'``, escaped once requests wraps it). +_URLLIB3_MISMATCH = re.compile(r"Fingerprints did not match.*?got \"(?:b\\?')?([0-9a-fA-F]{64})") + + +def fingerprint(der: bytes) -> str: + """``SHA256:AB:CD:…``: the SHA-256 of a DER certificate, as openssl prints it.""" + return _format(hashlib.sha256(der).hexdigest()) + + +def _format(hex_digest: str) -> str: + upper = hex_digest.upper() + return _PREFIX + ":".join(upper[i : i + 2] for i in range(0, len(upper), 2)) + + +def normalise_fingerprint(value: Any) -> str: + """*value* as :func:`fingerprint` writes it; ``SHA256:``, colons, spaces and case optional.""" + if not isinstance(value, str): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + text = value.strip() + if text[: len(_PREFIX)].upper() == _PREFIX: + text = text[len(_PREFIX) :] + digits = re.sub(r"[\s:]", "", text).upper() + if not _SHA256_HEX.match(digits): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + return _format(digits) + + +def _hex(fingerprint_: str) -> str: + """The bare lower-case hex urllib3's ``assert_fingerprint`` takes.""" + return fingerprint_.removeprefix(_PREFIX).replace(":", "").lower() + + +def fetch_certificate( + host: str, + port: int, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, +) -> bytes: + """The DER certificate *host* presents on *port*, from a handshake that sends nothing else. + + No CA and no host name are checked: the certificate is judged by its fingerprint. + """ + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with connect((host, port), timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as tls: + der = tls.getpeercert(binary_form=True) + if not der: # pragma: no cover - a TLS server always presents one + raise ConnectionException(f"{host}:{port} presented no TLS certificate") + return der + + +@dataclass(frozen=True) +class CertificateObservation: + """The certificate one connection saw, against the one it was to expect.""" + + host: str + port: int + #: The fingerprint of the certificate presented. + seen: str + #: The fingerprint it was to present; None on first use. + expected: str | None + + @property + def accepted(self) -> bool: + """True on first use and for the expected certificate; False for a refused one.""" + return self.expected is None or self.expected == self.seen + + def as_dict(self) -> dict[str, Any]: + return { + "host": self.host, + "port": self.port, + "seen": {"fingerprint": self.seen}, + "expected": {"fingerprint": self.expected} if self.expected else None, + "accepted": self.accepted, + } + + +class TlsCertificateMismatch(ConnectionException): # type: ignore[misc] + """The device presented another TLS certificate than the one it is known by. + + Raised before any request that carries credentials: nothing was sent to it. + """ + + def __init__(self, observation: CertificateObservation) -> None: + self.observation = observation + super().__init__( + f"TLS certificate mismatch at {observation.host}:{observation.port}: it presented " + f"{observation.seen}, the trusted certificate is {observation.expected or 'none'}. " + "Nothing was sent to it." + ) + + +def _expected(known: Any, port: int) -> str | None: + """The fingerprint in *known* (``{port: fingerprint}``, port as text or number) for *port*.""" + if not known: + return None + if not isinstance(known, Mapping): + raise ValueError(f"TLS fingerprints must be given by port, not as {type(known).__name__}") + value = known.get(str(port)) or known.get(port) + return normalise_fingerprint(value) if value else None + + +def _refused_fingerprint(exc: BaseException) -> str | None: + """The fingerprint urllib3 saw when it refused a pinned connection, in *exc*'s chain.""" + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + match = _URLLIB3_MISMATCH.search(str(current)) + if match: + return _format(match.group(1)) + stack.extend([current.__cause__, current.__context__]) + return None + + +class CertificateCheck: + """How one driver connection verifies its host's TLS certificate. + + *optional_args* are the driver's (both keys optional), *host* and *port* the + address the connection goes to. + """ + + def __init__(self, optional_args: Mapping[str, Any] | None, host: str, port: int) -> None: + args = optional_args or {} + self.host = host + self.port = int(port) + self.expected = _expected(args.get(KNOWN_FINGERPRINTS_ARG), self.port) + self._observer = args.get(OBSERVER_ARG) + self._verified: CertificateObservation | None = None + + def verify_server( + self, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, + ) -> CertificateObservation: + """Take the certificate from a handshake of its own and :meth:`verify` it.""" + return self.verify(fetch_certificate(self.host, self.port, timeout, connect=connect)) + + def verify(self, der: bytes) -> CertificateObservation: + """Judge the DER certificate *der*: a refused one is reported and raises. + + An accepted one is reported by :meth:`observe`, once the login stands. + """ + observation = CertificateObservation(self.host, self.port, fingerprint(der), self.expected) + if not observation.accepted: + self._report(observation) + raise TlsCertificateMismatch(observation) + self._verified = observation + return observation + + def pin(self, session: Any) -> None: + """Every HTTPS connection of the ``requests`` *session* must present the verified one.""" + verified = self._require_verified() + session.mount("https://", _pinned_adapter(_hex(verified.seen))) + + @contextmanager + def refusing(self) -> Iterator[None]: + """Around requests on a pinned session: a refused certificate becomes a mismatch.""" + try: + yield + except Exception as exc: + offered = _refused_fingerprint(exc) + if offered is None: + raise + trusted = self.expected or self._require_verified().seen + observation = CertificateObservation(self.host, self.port, offered, trusted) + self._report(observation) + raise TlsCertificateMismatch(observation) from exc + + def observe(self) -> CertificateObservation: + """After the login: the verified certificate, reported.""" + verified = self._require_verified() + self._report(verified) + return verified + + def _require_verified(self) -> CertificateObservation: + if self._verified is None: + raise RuntimeError("verify the certificate first (verify_server or verify)") + return self._verified + + def _report(self, observation: CertificateObservation) -> None: + if not callable(self._observer): + return + try: + self._observer(observation.as_dict()) + except Exception: # the store is the caller's; a connection does not hang on it + logger.warning( + "Could not report the TLS certificate of %s:%s", + self.host, + self.port, + exc_info=True, + ) + + +def _pinned_adapter(hex_digest: str) -> Any: + """A ``requests`` adapter whose connections, direct or through a proxy, must match *hex_digest*. + + ``requests`` is imported here: it comes with napalm, and a module that only + verifies a fingerprint does not need it loaded. + """ + from requests.adapters import HTTPAdapter + + class _PinnedAdapter(HTTPAdapter): # type: ignore[misc] + def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: + kwargs["assert_fingerprint"] = hex_digest + super().init_poolmanager(*args, **kwargs) + + def proxy_manager_for(self, proxy: str, **kwargs: Any) -> Any: + kwargs["assert_fingerprint"] = hex_digest + return super().proxy_manager_for(proxy, **kwargs) + + return _PinnedAdapter() diff --git a/pyproject.toml b/pyproject.toml index 06800cb..ee6a8e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ ] dependencies = [ "napalm>=5.1", # older releases import pkg_resources, which a 3.12 venv lacks + "requests>=2.25.0", "typing-extensions>=4.0", ] diff --git a/tests/test_tls_certificates.py b/tests/test_tls_certificates.py new file mode 100644 index 0000000..ab0f817 --- /dev/null +++ b/tests/test_tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +The layer above keeps the store (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +A driver gets the fingerprint it should expect through ``optional_args["tls_fingerprints"]`` +(by port) and reports every certificate it sees through +``optional_args["tls_certificate_observer"]``. The handshake tests run real TLS against an +in-process server on a socket pair: what matters is the certificate the server presents in +a real handshake, which a fake could not show. +""" + +from __future__ import annotations + +import datetime +import hashlib +import socket +import ssl +import threading + +import pytest +import requests +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID +from napalm.base.exceptions import ConnectionException + +from napalm_device_types.tls_certificates import ( + KNOWN_FINGERPRINTS_ARG, + OBSERVER_ARG, + CertificateCheck, + TlsCertificateMismatch, + fetch_certificate, + fingerprint, + normalise_fingerprint, +) + +DER = b"not really a certificate, but bytes are bytes" +DER_HEX = hashlib.sha256(DER).hexdigest().upper() +DER_FP = "SHA256:" + ":".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)) +OTHER_FP = "SHA256:" + ":".join(["AB"] * 32) + + +def _self_signed(tmp_path, name: str = "switch") -> tuple[bytes, str, str]: + """A self-signed certificate, as a ProCurve ships one: (DER, cert file, key file).""" + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + now = datetime.datetime.now(datetime.UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .sign(key, hashes.SHA256()) + ) + cert_file, key_file = tmp_path / f"{name}.crt", tmp_path / f"{name}.key" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + return cert.public_bytes(serialization.Encoding.DER), str(cert_file), str(key_file) + + +class _TlsServer: + """A TLS server on one end of a socket pair; it handshakes and then waits.""" + + def __init__(self, cert_file: str, key_file: str) -> None: + self.client_sock, server_sock = socket.socketpair() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_file, key_file) + self._server_sock = server_sock + self._context = context + self.received = b"" + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + try: + tls = self._context.wrap_socket(self._server_sock, server_side=True) + self.received = tls.recv(1024) + except (OSError, ssl.SSLError): # the client hung up after the handshake + pass + + def connect(self, address, timeout=None): + return self.client_sock + + def close(self) -> None: + self.client_sock.close() + self._server_sock.close() + self._thread.join(timeout=5) + + +@pytest.fixture +def certificate(tmp_path): + return _self_signed(tmp_path) + + +@pytest.fixture +def server(certificate): + _der, cert_file, key_file = certificate + s = _TlsServer(cert_file, key_file) + yield s + s.close() + + +def _check(expected: str | None, seen: list, port: int = 443) -> CertificateCheck: + args: dict = {OBSERVER_ARG: seen.append} + if expected is not None: + args[KNOWN_FINGERPRINTS_ARG] = {str(port): expected} + return CertificateCheck(args, "10.0.0.5", port) + + +class TestFingerprint: + def test_is_sha256_of_the_der_certificate_in_colon_hex(self): + assert fingerprint(DER) == DER_FP + + @pytest.mark.parametrize( + "value", + [ + DER_FP, + DER_FP.lower(), + DER_HEX, + DER_HEX.lower(), + DER_FP.removeprefix("SHA256:"), + " ".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)), + "sha256:" + DER_HEX, + ], + ) + def test_a_pin_may_be_written_as_openssl_or_a_browser_shows_it(self, value): + assert normalise_fingerprint(value) == DER_FP + + @pytest.mark.parametrize( + "value", ["", "AB:CD", "SHA1:" + DER_HEX, "ZZ" * 32, DER_HEX + "AB", 42] + ) + def test_anything_else_is_refused(self, value): + with pytest.raises(ValueError): + normalise_fingerprint(value) + + +class TestKnownFingerprints: + def test_the_pin_is_the_one_for_the_port_connected_to(self): + args = {KNOWN_FINGERPRINTS_ARG: {"443": DER_HEX}} + assert CertificateCheck(args, "10.0.0.5", 443).expected == DER_FP + assert CertificateCheck(args, "10.0.0.5", 8443).expected is None + + def test_a_port_may_be_given_as_a_number(self): + args = {KNOWN_FINGERPRINTS_ARG: {8443: DER_FP}} + assert CertificateCheck(args, "10.0.0.5", 8443).expected == DER_FP + + def test_a_malformed_mapping_is_refused(self): + with pytest.raises(ValueError): + CertificateCheck({KNOWN_FINGERPRINTS_ARG: [DER_FP]}, "10.0.0.5", 443) + + def test_without_optional_args_it_is_first_use(self): + assert CertificateCheck(None, "10.0.0.5", 443).expected is None + + +class TestVerify: + def test_first_use_accepts_and_reports_only_once_observed(self): + seen: list = [] + check = _check(None, seen) + observation = check.verify(DER) + assert observation.accepted is True + assert seen == [] + check.observe() + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": None, + "accepted": True, + } + ] + + def test_the_pinned_certificate_is_accepted(self): + seen: list = [] + check = _check(DER_HEX.lower(), seen) + assert check.verify(DER).accepted is True + check.observe() + assert seen[0]["expected"] == {"fingerprint": DER_FP} + assert seen[0]["accepted"] is True + + def test_another_certificate_is_refused_reported_and_named(self): + seen: list = [] + with pytest.raises(TlsCertificateMismatch) as caught: + _check(OTHER_FP, seen).verify(DER) + assert caught.value.observation.seen == DER_FP + assert caught.value.observation.expected == OTHER_FP + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": {"fingerprint": OTHER_FP}, + "accepted": False, + } + ] + + def test_the_mismatch_is_a_connection_failure_that_names_both(self): + with pytest.raises(ConnectionException) as caught: + _check(OTHER_FP, []).verify(DER) + message = str(caught.value) + assert "10.0.0.5:443" in message + assert DER_FP in message + assert OTHER_FP in message + + def test_a_failing_observer_does_not_break_the_connection(self): + def boom(_observation): + raise RuntimeError("store down") + + check = CertificateCheck({OBSERVER_ARG: boom}, "10.0.0.5", 443) + check.verify(DER) + check.observe() + + def test_observe_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).observe() + + +class TestHandshake: + def test_fetch_returns_the_certificate_the_server_presents(self, server, certificate): + der, _cert, _key = certificate + assert fetch_certificate("10.0.0.5", 443, 5, connect=server.connect) == der + + def test_verify_server_refuses_another_certificate_before_anything_is_sent( + self, server, tmp_path + ): + other_der, _c, _k = _self_signed(tmp_path, "impostor") + seen: list = [] + check = _check(fingerprint(other_der), seen) + with pytest.raises(TlsCertificateMismatch): + check.verify_server(5, connect=server.connect) + server.close() + assert server.received == b"" + assert seen[0]["accepted"] is False + + def test_verify_server_accepts_on_first_use(self, server, certificate): + der, _cert, _key = certificate + observation = _check(None, []).verify_server(5, connect=server.connect) + assert observation.seen == fingerprint(der) + + +class TestRequests: + def test_pin_makes_every_connection_present_the_verified_certificate(self): + check = _check(None, []) + check.verify(DER) + session = requests.Session() + check.pin(session) + adapter = session.get_adapter("https://10.0.0.5/rest/v7/") + assert adapter.poolmanager.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + proxy = adapter.proxy_manager_for("http://proxy.example:3128") + assert proxy.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + + def test_pin_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).pin(requests.Session()) + + def test_a_changed_certificate_on_a_pinned_session_is_a_mismatch(self): + from urllib3.util.ssl_ import assert_fingerprint + + seen: list = [] + check = _check(DER_FP, seen) + check.verify(DER) + impostor = b"another certificate" + with pytest.raises(TlsCertificateMismatch) as caught, check.refusing(): + try: + assert_fingerprint(impostor, DER_HEX) + except Exception as exc: + raise requests.exceptions.SSLError(exc) from exc + assert caught.value.observation.seen == fingerprint(impostor) + assert caught.value.observation.expected == DER_FP + assert seen[-1]["accepted"] is False + + def test_refusing_lets_other_errors_through(self): + check = _check(None, []) + check.verify(DER) + with pytest.raises(requests.exceptions.ConnectionError), check.refusing(): + raise requests.exceptions.ConnectionError("no route to host") From a6289cb24a82b25652270ab94972fa50b81923d4 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:15:01 +0100 Subject: [PATCH 45/50] feat: HA state, backups, quorum votes, migrate_vm and node maintenance get_clusters() may also report the cluster's HA resources, backup jobs, the newest backup per guest and storage, and each node's quorum votes (all NotRequired; None means not read). HypervisorDriver declares get_vm_migration_preflight(), migrate_vm() and set_node_maintenance(); the preflight shape lives in vm_migration.py. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761). --- CHANGELOG.md | 15 +++++++ napalm_device_types/__init__.py | 8 ++++ napalm_device_types/clusters.py | 42 +++++++++++++++++++ napalm_device_types/hypervisor.py | 35 ++++++++++++++++ napalm_device_types/vm_migration.py | 26 ++++++++++++ tests/test_clusters.py | 58 ++++++++++++++++++++++++- tests/test_vm_migration.py | 65 +++++++++++++++++++++++++++++ 7 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 napalm_device_types/vm_migration.py create mode 100644 tests/test_vm_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ad65a9..7f1879e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,21 @@ recorded here; their history is the git log. 3.12 venv no longer has. ### Added +- `HypervisorDriver.get_vm_migration_preflight(name)`, `migrate_vm(name, target_node, + online=True, target_storage=None, timeout=3600)` and `set_node_maintenance(node, + enabled)`: which nodes of its cluster a guest could move to and what blocks it + (`MigrationPreflightDict`: `allowed_nodes`, `refused_nodes` with the platform's + reason, `local_disks` the migration copies, `local_resources` that block it), the + move itself, blocking until the platform's task ended and raising with the + platform's own cause, and a node's HA maintenance mode. Declared only; netOrk + probes each with `hasattr`. +- `get_clusters()` may report HA, backups and quorum votes: `ClusterDict.ha_resources` + (`HAResourceDict`: requested and current state, node, PVE 8 group, PVE 9 rules), + `backup_jobs` (`BackupJobDict`: schedule, storage, mode, the guests a job covers) + and `backups` (`BackupDict`: per guest and storage the newest backup, its size + and how many there are), and `ClusterNodeDict.votes`. All `NotRequired`, so + drivers written before stay valid; None means could not be read, `[]` read and + empty. - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 743e0d5..31df183 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -88,10 +88,13 @@ from napalm_device_types.channel import ( run_on_transport, ) from napalm_device_types.clusters import ( + BackupDict, + BackupJobDict, ClusterDict, ClusterNetworkDict, ClusterNodeDict, ClusterStorageDict, + HAResourceDict, ) from napalm_device_types.config_lifecycle import ConfigLifecycleMixin from napalm_device_types.config_sync import ConfigSyncMixin @@ -238,13 +241,18 @@ from napalm_device_types.virtual_ips import ( VirtualIpsMixin, parse_virtual_ips, ) +from napalm_device_types.vm_migration import MigrationPreflightDict from napalm_device_types.vm_stop_request import VmStopRequestMixin __all__ = [ + "BackupDict", + "BackupJobDict", "ClusterDict", "ClusterNetworkDict", "ClusterNodeDict", "ClusterStorageDict", + "HAResourceDict", + "MigrationPreflightDict", "HypervisorHostDict", "DnsForwardDict", "DnsForwardMixin", diff --git a/napalm_device_types/clusters.py b/napalm_device_types/clusters.py index c5e136e..cbc77ab 100644 --- a/napalm_device_types/clusters.py +++ b/napalm_device_types/clusters.py @@ -21,6 +21,7 @@ class ClusterNodeDict(TypedDict): cpu_usage: float | None # 0.0–1.0 memory_total: int | None # bytes memory_used: int | None # bytes + votes: NotRequired[int | None] # quorum votes of this node; None standalone/unreadable class ClusterStorageDict(TypedDict): @@ -45,6 +46,43 @@ class ClusterNetworkDict(TypedDict): nodes: list[str] +class HAResourceDict(TypedDict): + """A guest the cluster's HA manager looks after.""" + + vmid: int + kind: str # "qemu" or "lxc" + requested: str # state HA is asked to keep: "started", "stopped", "disabled", "ignored" + status: str | None # what the HA manager reports now: "started", "fence", "migrate", ... + node: str | None # the node the HA manager places it on now + group: NotRequired[str | None] # HA group (Proxmox VE 8) + rules: list[str] # ids of the HA rules that name it (Proxmox VE 9); [] where there are none + + +class BackupJobDict(TypedDict): + """A scheduled backup job of the cluster.""" + + id: str + enabled: bool + schedule: str | None # as the platform writes it: "sat 02:00", "daily", "mon..fri 21:00" + storage: str | None + mode: str | None # "snapshot", "suspend", "stop" + all: bool # every guest, minus ``exclude`` + vmids: list[int] # the guests it names; a pool job: the pool's members now; [] with all + exclude: list[int] + pool: str | None + node: str | None # the one node it runs on; None for every node + + +class BackupDict(TypedDict): + """The backups of one guest on one storage.""" + + vmid: int + storage: str + time: int # unix seconds of the newest backup of this guest on this storage + size: int | None # bytes of that newest backup + count: int # backups of this guest on this storage + + class ClusterDict(TypedDict): """A cluster the connection can see. A standalone host is a cluster of one, with no name.""" @@ -55,3 +93,7 @@ class ClusterDict(TypedDict): nodes: list[ClusterNodeDict] storages: list[ClusterStorageDict] networks: list[ClusterNetworkDict] + # None: could not be read; []: read, nothing there. + ha_resources: NotRequired[list[HAResourceDict] | None] + backup_jobs: NotRequired[list[BackupJobDict] | None] + backups: NotRequired[list[BackupDict] | None] diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 3618c22..7b2a8d0 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -31,6 +31,7 @@ from napalm_device_types.models import ( ) from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents +from napalm_device_types.vm_migration import MigrationPreflightDict class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver): @@ -662,6 +663,40 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... + def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict: + """Which nodes of its cluster the guest could move to, and what blocks it + (see ``vm_migration``). + + :raises ValueError: If no VM with the given name/ID exists. + """ + ... + + def migrate_vm( + self, + name: str, + target_node: str, + online: bool = True, + target_storage: str | None = None, + timeout: int = 3600, + ) -> None: + """Move the guest to another node of its cluster; block until the + platform's task ended. ``online`` keeps a running guest running. + + :raises ValueError: Unknown VM, or a target that is not another + member of the cluster. + :raises RuntimeError: The migration failed, with the platform's cause. + """ + ... + + def set_node_maintenance(self, node: str, enabled: bool) -> None: + """Switch HA maintenance mode of a cluster node; HA guests leave it + while it is on. + + :raises ValueError: If ``node`` is not a member of the cluster. + :raises RuntimeError: If the platform refuses or cannot be asked. + """ + ... + def get_hypervisor_hosts(self) -> list[HypervisorHostDict]: """The hosts this hypervisor or its manager runs, and how each stands: connected, in maintenance, with configuration issues, under a central diff --git a/napalm_device_types/vm_migration.py b/napalm_device_types/vm_migration.py new file mode 100644 index 0000000..23d14c2 --- /dev/null +++ b/napalm_device_types/vm_migration.py @@ -0,0 +1,26 @@ +"""Moving a guest to another node of its cluster (``HypervisorDriver.migrate_vm``). + +``get_vm_migration_preflight`` reports what the platform says before a move: +which nodes would take the guest, which refuse it and why, and what the guest +holds on its current node. ``migrate_vm`` then moves it and blocks until the +platform's task ended; a failure raises with the platform's own cause. +``set_node_maintenance`` switches a node's HA maintenance mode, which makes the +HA manager move its HA guests away while it is on. + +Facts only: which node to pick, and whether a local disk is worth copying, is +the caller's to decide. +""" + +from typing import TypedDict + + +class MigrationPreflightDict(TypedDict): + """Where a guest could move to, and what stands in the way.""" + + running: bool + allowed_nodes: list[str] # nodes the platform would move it to + refused_nodes: dict[str, str] # node -> the platform's reason, in plain words + # volume ids on storage the target does not share; they block nothing by + # themselves, the migration copies them + local_disks: list[str] + local_resources: list[str] # e.g. "hostpci0", "usb0": they block the migration diff --git a/tests/test_clusters.py b/tests/test_clusters.py index 30e537c..d678669 100644 --- a/tests/test_clusters.py +++ b/tests/test_clusters.py @@ -23,10 +23,13 @@ from typing import NotRequired, get_type_hints import napalm_device_types.hypervisor as hypervisor_module from napalm_device_types import ( + BackupDict, + BackupJobDict, ClusterDict, ClusterNetworkDict, ClusterNodeDict, ClusterStorageDict, + HAResourceDict, HypervisorDriver, ) @@ -52,17 +55,26 @@ class TestClusterListing: class TestClusterShape: def test_cluster(self): - assert get_type_hints(ClusterDict) == { + assert get_type_hints(ClusterDict, include_extras=True) == { "id": str | None, "name": str | None, "quorate": bool | None, "nodes": list[ClusterNodeDict], "storages": list[ClusterStorageDict], "networks": list[ClusterNetworkDict], + "ha_resources": NotRequired[list[HAResourceDict] | None], + "backup_jobs": NotRequired[list[BackupJobDict] | None], + "backups": NotRequired[list[BackupDict] | None], } + def test_cluster_keys_added_later_stay_optional(self): + """A driver written before HA and backups were part of the report stays valid.""" + assert ClusterDict.__required_keys__ == frozenset( + {"id", "name", "quorate", "nodes", "storages", "networks"} + ) + def test_node(self): - hints = get_type_hints(ClusterNodeDict) + hints = get_type_hints(ClusterNodeDict, include_extras=True) assert hints == { "name": str, "status": str, @@ -75,7 +87,9 @@ class TestClusterShape: "cpu_usage": float | None, "memory_total": int | None, "memory_used": int | None, + "votes": NotRequired[int | None], } + assert "votes" not in ClusterNodeDict.__required_keys__ def test_storage(self): assert get_type_hints(ClusterStorageDict) == { @@ -102,6 +116,46 @@ class TestClusterShape: ) +class TestHAAndBackupShapes: + def test_ha_resource(self): + hints = get_type_hints(HAResourceDict, include_extras=True) + assert hints == { + "vmid": int, + "kind": str, + "requested": str, + "status": str | None, + "node": str | None, + "group": NotRequired[str | None], + "rules": list[str], + } + assert HAResourceDict.__required_keys__ == frozenset( + {"vmid", "kind", "requested", "status", "node", "rules"} + ) + + def test_backup_job(self): + assert get_type_hints(BackupJobDict) == { + "id": str, + "enabled": bool, + "schedule": str | None, + "storage": str | None, + "mode": str | None, + "all": bool, + "vmids": list[int], + "exclude": list[int], + "pool": str | None, + "node": str | None, + } + + def test_backup(self): + assert get_type_hints(BackupDict) == { + "vmid": int, + "storage": str, + "time": int, + "size": int | None, + "count": int, + } + + class TestCreateVmTakesANode: def test_node_is_an_optional_keyword(self): fn = _declared("create_vm_from_cloud_init") diff --git a/tests/test_vm_migration.py b/tests/test_vm_migration.py new file mode 100644 index 0000000..18d7a8a --- /dev/null +++ b/tests/test_vm_migration.py @@ -0,0 +1,65 @@ +"""Moving a guest to another node of its cluster, and a node's HA maintenance mode. + +``get_vm_migration_preflight`` says where a guest could go and what holds it +back, ``migrate_vm`` moves it and waits, ``set_node_maintenance`` switches a +node's HA maintenance mode. Declared only, under ``TYPE_CHECKING``: netOrk +probes each with ``hasattr`` and hides what a driver lacks (VMware has none of +them yet). +""" + +from __future__ import annotations + +import ast +import inspect +from typing import get_type_hints + +import pytest + +import napalm_device_types.hypervisor as hypervisor_module +from napalm_device_types import HypervisorDriver, MigrationPreflightDict + + +def _declared(name: str) -> ast.FunctionDef: + tree = ast.parse(inspect.getsource(hypervisor_module)) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == name: + return node + raise AssertionError(f"HypervisorDriver does not declare {name}()") + + +def _signature(name: str) -> str: + fn = _declared(name) + return f"({ast.unparse(fn.args)}) -> {ast.unparse(fn.returns) if fn.returns else None}" + + +class TestDeclarations: + @pytest.mark.parametrize( + "method", ["get_vm_migration_preflight", "migrate_vm", "set_node_maintenance"] + ) + def test_absent_until_a_driver_implements_it(self, method): + assert not hasattr(HypervisorDriver, method) + + def test_preflight(self): + assert _signature("get_vm_migration_preflight") == ( + "(self, name: str) -> MigrationPreflightDict" + ) + + def test_migrate(self): + assert _signature("migrate_vm") == ( + "(self, name: str, target_node: str, online: bool=True, " + "target_storage: str | None=None, timeout: int=3600) -> None" + ) + + def test_maintenance(self): + assert _signature("set_node_maintenance") == "(self, node: str, enabled: bool) -> None" + + +class TestPreflightShape: + def test_keys(self): + assert get_type_hints(MigrationPreflightDict) == { + "running": bool, + "allowed_nodes": list[str], + "refused_nodes": dict[str, str], + "local_disks": list[str], + "local_resources": list[str], + } From 73771aba006fcdf45c87f0302b0fed62d68011d8 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 23:02:46 +0100 Subject: [PATCH 46/50] docs: a tighter create_vm_from_cloud_init docstring The VMProvisionIncomplete entry pushed the method past its recorded length in the quality baseline. The image-cache note joins the paragraph it repeated, and the Raises entry takes two lines; nothing it says is dropped. --- napalm_device_types/hypervisor.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index a9df8ea..8784884 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -473,13 +473,11 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ Create a new virtual machine from a cloud image via Cloud-Init. - Downloads the cloud image (qcow2/raw) directly on the hypervisor if not - already cached there, creates a new VM shell, imports the image as its - root disk, configures virtual network interfaces, and injects Cloud-Init - configuration via a storage snippet or similar mechanism. The resulting - VM is left in a running state. - - Implementations cache downloaded images on the hypervisor by URL/filename. + Downloads the cloud image (qcow2/raw) directly on the hypervisor unless it is + cached there already (implementations cache by URL/filename), creates a new VM + shell, imports the image as its root disk, configures virtual network interfaces, + and injects Cloud-Init configuration via a storage snippet or similar mechanism. + The resulting VM is left in a running state. Args: name (string) - new VM display name @@ -533,9 +531,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri Raises: RuntimeError - if provisioning fails (download failure, checksum mismatch, storage unavailable, invalid config, timeout, etc.) - VMProvisionIncomplete - a RuntimeError for a failure after the VM - was created: it carries that VM's ``vmid`` and ``node``, so the - caller can ``destroy_vm`` it, and the failure as its cause. + VMProvisionIncomplete - a RuntimeError for a failure after the VM was created, + with that VM's ``vmid`` and ``node`` for ``destroy_vm`` and the failure as cause. """ ... From f4354f498eade775bba88bec1dcff68e0eba7be8 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 23:05:13 +0100 Subject: [PATCH 47/50] refactor: the SNMP health profile shapes in snmp_health_profile.py models.py is over the file size limit and may not grow; the unknown key pushed it past its recorded length. SnmpHealthProfile, SnmpValueSpec, SnmpMemorySpec and SnmpInterfacesSpec move to napalm_device_types/snmp_health_profile.py. The package exports them as before and models re-exports them, so existing imports keep working. The baseline records the smaller models.py. --- CHANGELOG.md | 4 ++ napalm_device_types/__init__.py | 10 ++-- napalm_device_types/health_metrics.py | 4 +- napalm_device_types/models.py | 58 +++------------------ napalm_device_types/snmp_health_profile.py | 60 ++++++++++++++++++++++ tests/quality_baseline.json | 2 +- tests/test_snmp_health_profile.py | 8 +++ 7 files changed, 87 insertions(+), 59 deletions(-) create mode 100644 napalm_device_types/snmp_health_profile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f06606..b02fcff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ recorded here; their history is the git log. on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). - The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python 3.12 venv no longer has. +- The SNMP health profile shapes (`SnmpHealthProfile`, `SnmpValueSpec`, + `SnmpMemorySpec`, `SnmpInterfacesSpec`) are defined in + `napalm_device_types.snmp_health_profile`. The package and `models` still export + them, so an existing import keeps working. ### Added - `HypervisorDriver.get_vm_migration_preflight(name)`, `migrate_vm(name, target_node, diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 31df183..3acdac1 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -163,10 +163,6 @@ from napalm_device_types.models import ( PowerAction, PowerStateDict, SnmpAgentSettingsDict, - SnmpHealthProfile, - SnmpInterfacesSpec, - SnmpMemorySpec, - SnmpValueSpec, UpdateProgressDict, ) from napalm_device_types.nat_vpn import NatVpnMixin @@ -195,6 +191,12 @@ from napalm_device_types.service_packages import ( parse_service_packages, ) from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.snmp_health_profile import ( + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, +) from napalm_device_types.ssh_host_keys import ( KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, ) diff --git a/napalm_device_types/health_metrics.py b/napalm_device_types/health_metrics.py index 5066475..de0bd45 100644 --- a/napalm_device_types/health_metrics.py +++ b/napalm_device_types/health_metrics.py @@ -2,8 +2,8 @@ **Deprecated** (docs/STRUCTURE.md): drivers do not collect over SNMP. The application polls; a driver whose device publishes values under its own OIDs -declares them as ``SNMP_HEALTH_PROFILE`` (:class:`~napalm_device_types.models.SnmpHealthProfile`). -This mixin goes in 5.0. +declares them as ``SNMP_HEALTH_PROFILE`` +(:class:`~napalm_device_types.snmp_health_profile.SnmpHealthProfile`). This mixin goes in 5.0. The collection itself is generic: CPU, memory, uptime, load and per-interface counters come from the same standard OIDs on any UCD-MIB/IF-MIB capable device. diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 86828ee..b791092 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -9,6 +9,12 @@ from typing import Literal, NotRequired, get_args from typing_extensions import TypedDict +# The SNMP health profile shapes live in snmp_health_profile.py; re-exported here, where they were. +from napalm_device_types.snmp_health_profile import SnmpHealthProfile as SnmpHealthProfile +from napalm_device_types.snmp_health_profile import SnmpInterfacesSpec as SnmpInterfacesSpec +from napalm_device_types.snmp_health_profile import SnmpMemorySpec as SnmpMemorySpec +from napalm_device_types.snmp_health_profile import SnmpValueSpec as SnmpValueSpec + # --------------------------------------------------------------------------- # Common (shared across device types) # --------------------------------------------------------------------------- @@ -1230,58 +1236,6 @@ class SnmpAgentSettingsDict(TypedDict): listen: NotRequired[str] # the address the agent binds to; absent: every address -class SnmpValueSpec(TypedDict, total=False): - """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. - - Either ``oids``, scalars asked in order (the first non-empty answer wins), or - ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a - regular expression whose first group is the number, for an agent that answers - with text. - - ``unknown`` says the agent answers this value wrongly, or not at all, where it - should: the application reports no value -- neither the standard reading nor a - default. It stands alone, without ``oids`` or ``table``. - """ - - oids: list[str] - table: str - fold: Literal["first", "average", "max"] - pattern: str - unknown: bool - - -class SnmpMemorySpec(TypedDict, total=False): - """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" - - total: SnmpValueSpec - used: SnmpValueSpec - free: SnmpValueSpec - percent: SnmpValueSpec - unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default - - -class SnmpInterfacesSpec(TypedDict, total=False): - """How the IF-MIB counters of a device are to be read.""" - - skip: str # a pattern of interface names whose counters are noise, not signal - tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) - - -class SnmpHealthProfile(TypedDict, total=False): - """Where a device's SNMP agent publishes its health values. - - A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer - the standard UCD-MIB and IF-MIB; the application polls and computes. What a - profile does not name is read the standard way, and what it marks ``unknown`` is - not read at all. Uptime is in TimeTicks, CPU in percent busy. - """ - - uptime: SnmpValueSpec - cpu: SnmpValueSpec - memory: SnmpMemorySpec - interfaces: SnmpInterfacesSpec - - class NICConfigDict(TypedDict): """Network interface configuration for VM provisioning.""" diff --git a/napalm_device_types/snmp_health_profile.py b/napalm_device_types/snmp_health_profile.py new file mode 100644 index 0000000..7a01d5f --- /dev/null +++ b/napalm_device_types/snmp_health_profile.py @@ -0,0 +1,60 @@ +"""The shape a driver states its device's SNMP health OIDs in (``SNMP_HEALTH_PROFILE``). + +Apart from ``models.py``, which is over the file size limit; ``models`` still exports them. +""" + +from typing import Literal + +from typing_extensions import TypedDict + + +class SnmpValueSpec(TypedDict, total=False): + """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. + + Either ``oids``, scalars asked in order (the first non-empty answer wins), or + ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a + regular expression whose first group is the number, for an agent that answers + with text. + + ``unknown`` says the agent answers this value wrongly, or not at all, where it + should: the application reports no value -- neither the standard reading nor a + default. It stands alone, without ``oids`` or ``table``. + """ + + oids: list[str] + table: str + fold: Literal["first", "average", "max"] + pattern: str + unknown: bool + + +class SnmpMemorySpec(TypedDict, total=False): + """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" + + total: SnmpValueSpec + used: SnmpValueSpec + free: SnmpValueSpec + percent: SnmpValueSpec + unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default + + +class SnmpInterfacesSpec(TypedDict, total=False): + """How the IF-MIB counters of a device are to be read.""" + + skip: str # a pattern of interface names whose counters are noise, not signal + tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) + + +class SnmpHealthProfile(TypedDict, total=False): + """Where a device's SNMP agent publishes its health values. + + A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer + the standard UCD-MIB and IF-MIB; the application polls and computes. What a + profile does not name is read the standard way, and what it marks ``unknown`` is + not read at all. Uptime is in TimeTicks, CPU in percent busy. + """ + + uptime: SnmpValueSpec + cpu: SnmpValueSpec + memory: SnmpMemorySpec + interfaces: SnmpInterfacesSpec diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 82c402c..3b89766 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -32,7 +32,7 @@ }, "mypy_relaxed": [], "python_file_lines": { - "napalm_device_types/models.py": 1504 + "napalm_device_types/models.py": 1463 }, "untyped_public": [] } diff --git a/tests/test_snmp_health_profile.py b/tests/test_snmp_health_profile.py index c0f8054..b4f2a10 100644 --- a/tests/test_snmp_health_profile.py +++ b/tests/test_snmp_health_profile.py @@ -52,6 +52,14 @@ def test_a_value_the_agent_gets_wrong_is_declared_unknown(): assert Driver.SNMP_HEALTH_PROFILE["cpu"].get("unknown") is True +def test_the_shapes_live_in_their_own_module_and_models_still_exports_them(): + from napalm_device_types import models, snmp_health_profile + + for name in ("SnmpHealthProfile", "SnmpValueSpec", "SnmpMemorySpec", "SnmpInterfacesSpec"): + assert getattr(models, name) is getattr(snmp_health_profile, name), name + assert getattr(snmp_health_profile, name).__module__ == snmp_health_profile.__name__ + + def test_the_collector_in_the_base_package_warns_that_it_is_deprecated(): async def get(oid): return None From 63039ce7586f15ca09f83e982d9fcfd60fc62061 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 2 Nov 2026 00:15:55 +0100 Subject: [PATCH 48/50] fix(types): the adapter's ignore holds with and without requests' stubs CI installs types-requests, so HTTPAdapter is typed there and the ignore for subclassing an untyped base is unused; locally it is needed. --- napalm_device_types/tls_certificates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/napalm_device_types/tls_certificates.py b/napalm_device_types/tls_certificates.py index d6e315e..6169383 100644 --- a/napalm_device_types/tls_certificates.py +++ b/napalm_device_types/tls_certificates.py @@ -273,7 +273,7 @@ def _pinned_adapter(hex_digest: str) -> Any: """ from requests.adapters import HTTPAdapter - class _PinnedAdapter(HTTPAdapter): # type: ignore[misc] + class _PinnedAdapter(HTTPAdapter): # type: ignore[misc,unused-ignore] def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: kwargs["assert_fingerprint"] = hex_digest super().init_poolmanager(*args, **kwargs) From 36cf117b95dd2d7ff7bd8598a89988980a7141cd Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 2 Nov 2026 00:21:38 +0100 Subject: [PATCH 49/50] chore(release): 4.23.0 --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f784d3d..bf010e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ recorded here; their history is the git log. ## [Unreleased] +## [4.23.0] - 2026-11-02 + ### Changed - Python 3.12 is the floor (`requires-python = ">=3.12"`): netOrk runs every driver on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). diff --git a/pyproject.toml b/pyproject.toml index ee6a8e7..0039557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.22.0" +version = "4.23.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.12" From f337ae7fb895552655a67d8f0d93a865605882e5 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 2 Nov 2026 08:07:48 +0100 Subject: [PATCH 50/50] fix(updates): read dnf's security advisories from the cached metadata DNF_SECURITY_COMMAND ran `dnf updateinfo list --security`, which downloads the repositories' metadata first whenever the cached copy has expired. On an Oracle Linux host on OCI that took longer than netOrk's whole 120 s poll. It now runs with --cacheonly, as APT_UPGRADABLE_COMMAND reads apt's local index; refreshing stays `dnf makecache`. Without a cache dnf fails at once, and the __DNF_RC marker already turns that into "security unknown". Closes #76 --- CHANGELOG.md | 8 ++++++++ napalm_device_types/package_updates.py | 7 +++++-- tests/test_package_updates.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf010e5..aa6e349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ recorded here; their history is the git log. ## [Unreleased] +### Fixed +- `DNF_SECURITY_COMMAND` reads the cached metadata only (`dnf --cacheonly`), as + `APT_UPGRADABLE_COMMAND` reads apt's local index (#76). With an expired cache dnf + downloaded the repositories' metadata first, and on an Oracle Linux host on OCI that + outlasted netOrk's whole poll budget. Refreshing the index stays a step of its own + (`dnf makecache`). Without a cache dnf fails at once, which already reads as + "security unknown". + ## [4.23.0] - 2026-11-02 ### Changed diff --git a/napalm_device_types/package_updates.py b/napalm_device_types/package_updates.py index 7ff7f9f..2d1b69f 100644 --- a/napalm_device_types/package_updates.py +++ b/napalm_device_types/package_updates.py @@ -28,9 +28,12 @@ APT_UPGRADABLE_COMMAND = "{ LC_ALL=C apt list --upgradable 2>/dev/null; echo __A #: Read-only. Lists the packages that a pending security advisory covers. A #: failed dnf prints nothing, the same as no advisories, so its exit status is -#: printed inside the group, as apt's is. +#: printed inside the group, as apt's is. It reads the cached metadata only, as +#: apt reads its local index: an expired cache otherwise makes dnf download the +#: repositories' metadata first, which can take minutes (#76). DNF_SECURITY_COMMAND = ( - "{ LC_ALL=C dnf updateinfo list --security --quiet 2>/dev/null; echo __DNF_RC=$?; } | cat" + "{ LC_ALL=C dnf --cacheonly updateinfo list --security --quiet 2>/dev/null;" + " echo __DNF_RC=$?; } | cat" ) # openssl/noble-updates,noble-security 3.0.13-0ubuntu3.6 amd64 [upgradable from: 3.0.13-0ubuntu3.5] diff --git a/tests/test_package_updates.py b/tests/test_package_updates.py index 63d1342..b12a796 100644 --- a/tests/test_package_updates.py +++ b/tests/test_package_updates.py @@ -176,6 +176,18 @@ __DNF_RC=0 with pytest.raises(ValueError, match="exit status 1"): parse_dnf_security(out) + def test_the_command_reads_only_the_cached_metadata(self, tmp_path): + """An expired cache made dnf download the repositories' metadata inside the + poll, which outlasted its whole budget (#76).""" + args = tmp_path / "args" + dnf = tmp_path / "dnf" + dnf.write_text(f"#!/bin/sh\nprintf '%s\\n' \"$@\" > {args}\n") + dnf.chmod(0o755) + env = {"PATH": f"{tmp_path}:{os.environ.get('PATH', '/usr/bin:/bin')}"} + subprocess.run(["sh", "-c", DNF_SECURITY_COMMAND], capture_output=True, env=env) + + assert "--cacheonly" in args.read_text().split() + @pytest.mark.parametrize( ("nevra", "name"), [