From d142db73e544b7d405217d65df7371259032dda6 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:43:09 +0100 Subject: [PATCH 01/14] fix(sdn): restore the virtual interface prefixes for set_vlan b1ba991 removed _VIRTUAL_IFACE_PREFIXES while _is_physical_uplink still read it. The AttributeError was swallowed by _find_switch_uplink, so set_vlan never found an uplink on any node. Restore the constant and cover both helpers with tests. Closes #37 --- CHANGELOG.md | 2 ++ napalm_proxmox/sdn_mixin.py | 11 +++++++ tests/test_switch_uplink.py | 58 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 tests/test_switch_uplink.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ff980..0fd0382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `get_ipv6_neighbors_table()` reports an unknown age as `-1.0`, as `get_arp_table()` already did (napalm-device-types' `ARP_AGE_UNKNOWN`), instead of `0.0`, which means "confirmed just now" (NetOrk/netork#1327). +- `set_vlan()` finds the node's switch uplink again. Since 0.8.0 every call failed + with "no LLDP-detected switch uplink found", whatever LLDP reported (#37). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/sdn_mixin.py b/napalm_proxmox/sdn_mixin.py index 80bd327..7568528 100644 --- a/napalm_proxmox/sdn_mixin.py +++ b/napalm_proxmox/sdn_mixin.py @@ -33,6 +33,17 @@ _JsonDict = dict[str, Any] class ProxmoxSDNMixin: """Mixin providing SDN, VLAN and network-instance NAPALM methods.""" + #: Name prefixes of Proxmox-internal and runtime virtual interfaces, never a switch uplink. + _VIRTUAL_IFACE_PREFIXES: tuple[str, ...] = ( + "fwpr", # Proxmox firewall proxy veth + "fwln", # Proxmox firewall link veth + "tap", # VM tap devices + "veth", # generic veth pairs (containers) + "virbr", # libvirt bridges + "docker", # Docker virtual interfaces + "lxcbr", # LXC bridges + ) + def _get_sdn_zones(self) -> list[_JsonDict]: try: return self._api.cluster.sdn.zones.get() or [] # type: ignore[union-attr] diff --git a/tests/test_switch_uplink.py b/tests/test_switch_uplink.py new file mode 100644 index 0000000..d33a68d --- /dev/null +++ b/tests/test_switch_uplink.py @@ -0,0 +1,58 @@ +"""The physical port set_vlan configures: _find_switch_uplink / _is_physical_uplink.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +from napalm_proxmox.sdn_mixin import ProxmoxSDNMixin + +NETWORK = [ + {"iface": "eno1", "type": "eth"}, + {"iface": "vmbr0", "type": "bridge", "bridge_ports": "eno1"}, +] + + +def _mixin(detail: dict, basic: dict) -> ProxmoxSDNMixin: + m = ProxmoxSDNMixin() + m._get_node_network = MagicMock(return_value=NETWORK) + m.get_lldp_neighbors_detail = MagicMock(return_value=detail) + m.get_lldp_neighbors = MagicMock(return_value=basic) + return m + + +class TestFindSwitchUplink: + def test_physical_port_with_a_bridge_neighbour_is_the_uplink(self): + m = _mixin( + detail={ + "tap100i0": [{"remote_system_capab": ["bridge"]}], + "eno1": [{"remote_system_capab": ["bridge", "router"]}], + }, + basic={}, + ) + assert m._find_switch_uplink() == "eno1" + + def test_basic_lldp_fallback_skips_a_virtual_port(self): + m = _mixin( + detail={}, + basic={"fwpr100p0": [{"hostname": "sw"}], "eno1": [{"hostname": "sw"}]}, + ) + assert m._find_switch_uplink() == "eno1" + + def test_no_physical_neighbour_is_none(self): + m = _mixin(detail={"tap100i0": [{"remote_system_capab": ["bridge"]}]}, basic={}) + assert m._find_switch_uplink() is None + + +class TestIsPhysicalUplink: + def test_eth_in_the_network_config_is_physical(self): + assert ProxmoxSDNMixin()._is_physical_uplink("eno1", {"eno1": {"type": "eth"}}) + + def test_virtual_prefix_is_never_physical(self): + network = {"tap100i0": {"type": ""}} + assert not ProxmoxSDNMixin()._is_physical_uplink("tap100i0", network) + + def test_port_missing_from_the_network_config_is_not_physical(self): + assert not ProxmoxSDNMixin()._is_physical_uplink("eno9", {}) + + def test_bridge_is_not_physical(self): + assert not ProxmoxSDNMixin()._is_physical_uplink("vmbr0", {"vmbr0": {"type": "bridge"}}) From 52d63eb68951391c50d21d81ece747fa6238160d Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:44:23 +0100 Subject: [PATCH 02/14] fix(snmp): let the snmpd.conf write report its exit status The write ended in `exit $s`, which ends the shell inside the braces _rc wraps around every command, so __PVE_RC was never printed and the step always failed. The write now shares the apt file writer, which ends in a test instead of exit. Closes #30 --- CHANGELOG.md | 3 +++ napalm_proxmox/system_mixin.py | 16 ++++++++-------- tests/test_fix_templates.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fd0382..b6d37d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 means "confirmed just now" (NetOrk/netork#1327). - `set_vlan()` finds the node's switch uplink again. Since 0.8.0 every call failed with "no LLDP-detected switch uplink found", whatever LLDP reported (#37). +- `configure_snmp_agent()` completes: its "configure" step reported a failure + with empty output although `/etc/snmp/snmpd.conf` was written, so the firewall + and start steps never ran (#30). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index 0ce4cb6..fa8e74e 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -45,6 +45,8 @@ _SUBSCRIPTION_STATUS = { _PACKAGE_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.+:~-]*$") #: What may stand in an snmpd.conf value: no whitespace, no quote, no comment. _SNMP_WORD = re.compile(r'^[^\s"#]+$') +#: The net-snmp agent's configuration file. +_SNMPD_CONF = "/etc/snmp/snmpd.conf" def _net(value: str) -> str: @@ -527,8 +529,12 @@ class ProxmoxSystemMixin: return self._step(command, done) def _write_apt_file(self, path: str, content: str) -> dict: + return self._write_root_file(path, content) + + def _write_root_file(self, path: str, content: str) -> dict: + """Write *content* to *path* as root, mode 644, through a temporary file.""" encoded = base64.b64encode(content.encode()).decode() - # No ``exit`` here: ``_rc`` reads the status after the command's braces. + # No ``exit`` here: it would end the shell before ``_rc`` reads the status. return self._step( f'umask 077 && t=$(mktemp) && echo {encoded} | base64 -d > "$t" && ' f'install -m 644 -o root -g root "$t" {shlex.quote(path)}; ' @@ -559,13 +565,7 @@ class ProxmoxSystemMixin: ) def _write_snmp_agent_config(self, settings: dict) -> dict: - encoded = base64.b64encode(render_snmpd_conf(settings).encode()).decode() - return self._step( - f'umask 077 && t=$(mktemp) && echo {encoded} | base64 -d > "$t" && ' - f'install -m 644 -o root -g root "$t" /etc/snmp/snmpd.conf; ' - 's=$?; rm -f "$t"; exit $s', - "wrote /etc/snmp/snmpd.conf", - ) + return self._write_root_file(_SNMPD_CONF, render_snmpd_conf(settings)) def _allow_snmp_sources(self, settings: dict) -> dict: sources = [_net(s) for s in settings.get("allowed_sources", [])] diff --git a/tests/test_fix_templates.py b/tests/test_fix_templates.py index f72021f..09511d5 100644 --- a/tests/test_fix_templates.py +++ b/tests/test_fix_templates.py @@ -10,6 +10,7 @@ contract's ``install_package``. Every step reads its exit status. from __future__ import annotations import re +import subprocess import pytest @@ -36,6 +37,18 @@ class _Node: return f"{out}\n__PVE_RC={rc}" if "__PVE_RC" in command else out +def _shell_driver(prelude: str) -> ProxmoxDriver: + """A driver whose node shell is a real local ``sh``, *prelude* run first.""" + + def shell(command: str) -> str: + run = subprocess.run(["sh", "-c", prelude + command], capture_output=True, text=True) + return run.stdout + + d = ProxmoxDriver("pve", "root@pam", "pw") + d._exec_ssh_command = shell + return d + + @pytest.fixture def node(): return _Node() @@ -87,6 +100,27 @@ class TestSnmpAgent: assert result["success"] is False assert not any("/etc/snmp/snmpd.conf" in c for c in node.ran) + def test_the_written_config_reports_success_through_a_real_shell(self, tmp_path): + """The configure step's own exit status must reach ``_rc`` (#30). + + ``_Node`` answers ``__PVE_RC=0`` on its own; a real ``sh`` shows whether the + wrapped command still prints it. ``install`` is a function writing to tmp_path. + """ + target = tmp_path / "snmpd.conf" + d = _shell_driver(f'install() {{ cat "$7" > {target}; }}; ') + + step = d._write_snmp_agent_config(SETTINGS) + + assert step == {"ok": True, "output": "wrote /etc/snmp/snmpd.conf"} + assert "rocommunity c0mm 10.7.224.20/32" in target.read_text() + + def test_a_refused_write_reports_failure_through_a_real_shell(self): + d = _shell_driver('install() { echo "install: read-only file system"; return 1; }; ') + + step = d._write_snmp_agent_config(SETTINGS) + + assert step == {"ok": False, "output": "install: read-only file system"} + def test_the_management_peer(self, driver): assert driver.get_management_peer() == "10.7.224.20" From 07796953496e0dfa5a3faa8f8daf5e7f6dd132e1 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:45:49 +0100 Subject: [PATCH 03/14] fix(vm-provision): pass every SSH key, one per line Proxmox's sshkeys holds one OpenSSH key per line, URL-encoded. The keys were joined with ";", so only the first one worked, and quote() kept "/" unescaped, which Proxmox's urlencoded format refuses. Closes #23 --- CHANGELOG.md | 3 ++ napalm_proxmox/vm_provision_mixin.py | 6 ++-- tests/test_vm_provision_mixin.py | 48 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6d37d0..96c89c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `configure_snmp_agent()` completes: its "configure" step reported a failure with empty output although `/etc/snmp/snmpd.conf` was written, so the firewall and start steps never ran (#30). +- `create_vm_from_cloud_init(ssh_public_keys=...)` installs every key, one per + line, instead of only the first; a key containing `/` or `+` is no longer + refused by Proxmox (#23). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/vm_provision_mixin.py b/napalm_proxmox/vm_provision_mixin.py index 9a2d206..0d7577d 100644 --- a/napalm_proxmox/vm_provision_mixin.py +++ b/napalm_proxmox/vm_provision_mixin.py @@ -736,9 +736,9 @@ class ProxmoxVMProvisionMixin: cloud_init_args[f"ipconfig{i}"] = "ip=dhcp" if ssh_public_keys: - # URL-encode SSH keys for Proxmox API - sshkeys = ";".join(ssh_public_keys) - cloud_init_args["sshkeys"] = quote(sshkeys) + # One key per line, URL-encoded without exception: Proxmox's + # "urlencoded" format refuses an unescaped "/" or "+". + cloud_init_args["sshkeys"] = quote("\n".join(ssh_public_keys), safe="") self._node_api().qemu(vmid).config.post(**cloud_init_args) diff --git a/tests/test_vm_provision_mixin.py b/tests/test_vm_provision_mixin.py index d3d9f0d..06fbe36 100644 --- a/tests/test_vm_provision_mixin.py +++ b/tests/test_vm_provision_mixin.py @@ -1280,3 +1280,51 @@ def test_create_vm_cloudinit_drive_uses_image_storage_not_snippet_storage(): cloud_init_call = next(c for c in mock_vm.config.post.call_args_list if "ide2" in c[1]) assert cloud_init_call[1]["ide2"] == "local-zfs:cloudinit" assert cloud_init_call[1]["cicustom"].startswith("user=local:snippets/") + + +def test_create_vm_passes_every_ssh_key_one_per_line(): + """Proxmox's sshkeys is one key per line, URL-encoded (napalm-proxmox#23). + + Joined with ";", only the first key worked. Base64 keys carry "/" and "+", + which Proxmox's urlencoded format refuses unescaped. + """ + import re + from urllib.parse import unquote + + mixin = ProxmoxVMProvisionMixin() + mixin._node_name = "pve1" + mock_api = MagicMock() + mock_api.cluster.nextid.get.return_value = 110 + mock_api.storage.return_value.get.return_value = {"path": "/var/lib/vz"} + mock_node = MagicMock() + mock_node.storage.get.return_value = [ + {"storage": "local", "type": "dir", "content": "images,snippets", "enabled": 1}, + ] + mixin._api = mock_api + mixin._node_api = MagicMock(return_value=mock_node) + mixin._download_cloud_image = MagicMock(return_value="/var/lib/vz/x.qcow2") + mixin._run_node_command = MagicMock(return_value="") + mock_vm = mock_node.qemu.return_value + mock_vm.config.get.return_value = {"unused0": "local:vm-110-disk-0"} + mock_vm.status.start.post.return_value = "UPID:pve1:136:start" + mock_node.tasks.return_value.status.get.return_value = {"status": "stopped", "exitstatus": "OK"} + keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI/+k1 alice@example", + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQAB/+k2 bob@example", + ] + + with patch("time.sleep"): + mixin.create_vm_from_cloud_init( + name="keys-vm", + image_url="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2", + cpu=1, + memory=512, + nics=[{"bridge": "vmbr0"}], + cloud_init_config={}, + ssh_public_keys=keys, + ) + + call = next(c for c in mock_vm.config.post.call_args_list if "sshkeys" in c[1]) + sshkeys = call[1]["sshkeys"] + assert unquote(sshkeys) == "\n".join(keys) + assert re.fullmatch(r"[-%a-zA-Z0-9_.!~*'()]*", sshkeys) From 89c70d85be165b79ee33b20c2deb5e26368f5e18 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:46:51 +0100 Subject: [PATCH 04/14] fix(lldp): keep every neighbour of a port apart lldpcli prints one Interface block per neighbour. The summary parser reused the port's last entry, so a second neighbour overwrote the first, and a neighbour without SysName was dropped. Each block now starts an entry of its own. Closes #22 --- CHANGELOG.md | 3 ++ napalm_proxmox/lldp_mixin.py | 24 ++++++------- tests/test_lldp.py | 65 ++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 tests/test_lldp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 96c89c7..a640423 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `create_vm_from_cloud_init(ssh_public_keys=...)` installs every key, one per line, instead of only the first; a key containing `/` or `+` is no longer refused by Proxmox (#23). +- `get_lldp_neighbors()` lists every neighbour of a port: a second neighbour on + the same port no longer overwrites the first, and a neighbour that sends no + system name is listed with an empty `hostname` instead of being dropped (#22). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/lldp_mixin.py b/napalm_proxmox/lldp_mixin.py index cd10613..c8eae80 100644 --- a/napalm_proxmox/lldp_mixin.py +++ b/napalm_proxmox/lldp_mixin.py @@ -26,26 +26,26 @@ class ProxmoxLLDPMixin: """Mixin providing LLDP NAPALM methods.""" def get_lldp_neighbors(self) -> dict[str, list[_JsonDict]]: - """Return LLDP neighbours (requires lldpd on the Proxmox node).""" + """Return LLDP neighbours (requires lldpd on the Proxmox node). + + lldpcli prints one ``Interface:`` block per neighbour, so a port with + several neighbours appears once for each of them. + """ result: dict[str, list[_JsonDict]] = {} raw = self._exec_ssh_command("lldpcli show neighbors summary 2>/dev/null || true") - current_iface = "" + current: _JsonDict | None = None for line in raw.splitlines(): m_iface = re.match(r"^\s*Interface:\s+(\S+?),?\s", line) if m_iface: - current_iface = m_iface.group(1) - result.setdefault(current_iface, []) + current = {"hostname": "", "port": ""} + result.setdefault(m_iface.group(1), []).append(current) continue m_sys = re.match(r"^\s*SysName:\s+(.+)", line) m_port = re.match(r"^\s*PortID:\s+\S+\s+(.+)", line) - if m_sys and current_iface: - hostname = m_sys.group(1).strip() - if result[current_iface]: - result[current_iface][-1]["hostname"] = hostname - else: - result[current_iface].append({"hostname": hostname, "port": ""}) - if m_port and current_iface and result[current_iface]: - result[current_iface][-1]["port"] = m_port.group(1).strip() + if m_sys and current is not None: + current["hostname"] = m_sys.group(1).strip() + if m_port and current is not None: + current["port"] = m_port.group(1).strip() return result def get_lldp_neighbors_detail(self, interface: str = "") -> dict[str, list[_JsonDict]]: diff --git a/tests/test_lldp.py b/tests/test_lldp.py new file mode 100644 index 0000000..f48ab62 --- /dev/null +++ b/tests/test_lldp.py @@ -0,0 +1,65 @@ +"""LLDP neighbours read from lldpd's ``lldpcli show neighbors``.""" + +from __future__ import annotations + +from napalm_proxmox import ProxmoxDriver + +# lldpd 1.0 output: one "Interface:" block per neighbour, so a port with two +# neighbours (a switch and a phone behind it) appears twice. The phone sends +# no SysName. +TWO_NEIGHBOURS_ON_ONE_PORT = """\ +------------------------------------------------------------------------------- +LLDP neighbors: +------------------------------------------------------------------------------- +Interface: eno1, via: LLDP, RID: 1, Time: 0 day, 00:10:12 + Chassis: + ChassisID: mac 00:11:22:33:44:55 + SysName: sw1 + Port: + PortID: ifname gi1/0/1 + PortDescr: GigabitEthernet1/0/1 +------------------------------------------------------------------------------- +Interface: eno1, via: LLDP, RID: 2, Time: 0 day, 00:01:02 + Chassis: + ChassisID: mac aa:bb:cc:dd:ee:ff + Port: + PortID: mac aa:bb:cc:dd:ee:ff +------------------------------------------------------------------------------- +Interface: eno2, via: LLDP, RID: 3, Time: 0 day, 00:10:12 + Chassis: + ChassisID: mac 00:11:22:33:44:66 + SysName: sw2 + Port: + PortID: ifname gi1/0/2 +------------------------------------------------------------------------------- +""" + + +def _driver(output: str) -> ProxmoxDriver: + d = ProxmoxDriver("pve", "root@pam", "pw") + d._exec_ssh_command = lambda command: output + return d + + +class TestGetLldpNeighbors: + def test_every_neighbour_on_a_port_is_its_own_entry(self): + result = _driver(TWO_NEIGHBOURS_ON_ONE_PORT).get_lldp_neighbors() + + assert result == { + "eno1": [ + {"hostname": "sw1", "port": "gi1/0/1"}, + {"hostname": "", "port": "aa:bb:cc:dd:ee:ff"}, + ], + "eno2": [{"hostname": "sw2", "port": "gi1/0/2"}], + } + + def test_no_lldpd_output_is_no_neighbour(self): + assert _driver("").get_lldp_neighbors() == {} + + +class TestGetLldpNeighborsDetail: + def test_every_neighbour_on_a_port_is_its_own_entry(self): + result = _driver(TWO_NEIGHBOURS_ON_ONE_PORT).get_lldp_neighbors_detail() + + assert [e["remote_system_name"] for e in result["eno1"]] == ["sw1", ""] + assert result["eno1"][1]["remote_chassis_id"] == "aa:bb:cc:dd:ee:ff" From 8f10bef7c5599ed810b39411a384c9a62e3d1344 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:47:59 +0100 Subject: [PATCH 05/14] fix(hostname): rename only the node's own names in /etc/hosts The rename replaced \b\b, which also matches before "." and "-": "pve.old.lan pve" became "new.dom.old.lan new.dom" and "pve-backup" became "new-backup". Whole whitespace-separated names are renamed now, comments left alone. The live move of /etc/pve/nodes/ the issue also asks about is not changed here; it needs a decision on clustered nodes. Refs #21 --- CHANGELOG.md | 3 ++ napalm_proxmox/system_mixin.py | 36 +++++++++++------ tests/quality_baseline.json | 2 +- tests/test_set_hostname.py | 72 ++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 tests/test_set_hostname.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a640423..cd9e856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `get_lldp_neighbors()` lists every neighbour of a port: a second neighbour on the same port no longer overwrites the first, and a neighbour that sends no system name is listed with an empty `hostname` instead of being dropped (#22). +- `set_hostname()` renames only the node's own names in `/etc/hosts`: other + hosts whose names start with the old one (`pve-backup`) and comments stay as + they are, and the old FQDN becomes the new one instead of a mix of both (#21). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index fa8e74e..7f63931 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -49,6 +49,30 @@ _SNMP_WORD = re.compile(r'^[^\s"#]+$') _SNMPD_CONF = "/etc/snmp/snmpd.conf" +def _rename_in_hosts(hosts: str, old_short: str, new_short: str, new_domain: str | None) -> str: + """*hosts* (an /etc/hosts) with the node's names renamed from *old_short*. + + Only whole names change: ``old_short`` becomes ``new_short``, and + ``old_short.`` becomes ``new_short.`` (the old domain + kept when *new_domain* is None). Other names that merely start with the + old name (``pve-backup``) and comments stay as they are. + """ + + def rename(match: re.Match[str]) -> str: + name = match.group(0) + if name == old_short: + return new_short + if name.startswith(f"{old_short}."): + return f"{new_short}.{new_domain or name[len(old_short) + 1 :]}" + return name + + lines = [] + for line in hosts.splitlines(keepends=True): + entry, hash_, comment = line.partition("#") + lines.append(re.sub(r"\S+", rename, entry) + hash_ + comment) + return "".join(lines) + + def _net(value: str) -> str: return str(ipaddress.ip_network(value.strip(), strict=False)) @@ -633,17 +657,7 @@ class ProxmoxSystemMixin: # write back via base64 to avoid shell-escaping issues. if old_short and old_short != short_hostname: hosts_raw = self._exec_ssh_command("cat /etc/hosts 2>/dev/null") - # Replace whole-word occurrences of the old short hostname. - # Both "hostname" and "hostname.domain" forms are covered because - # the new value is the full fqdn when a domain was given. - import re as _re - - new_replace = fqdn if domain else short_hostname - hosts_new = _re.sub( - r"\b" + _re.escape(old_short) + r"\b", - new_replace, - hosts_raw, - ) + hosts_new = _rename_in_hosts(hosts_raw, old_short, short_hostname, domain) hosts_b64 = _b64.b64encode(hosts_new.encode()).decode() self._exec_ssh_command( f"echo {hosts_b64} | base64 -d > /tmp/_netork_hosts" diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 75fd5b4..17d19e2 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -36,7 +36,7 @@ "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_disk_smart": 74, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_packages": 66, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_users": 51, - "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.set_hostname": 124, + "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.set_hostname": 114, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vm_interfaces": 179, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vms": 114, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin._download_cloud_image": 61, diff --git a/tests/test_set_hostname.py b/tests/test_set_hostname.py new file mode 100644 index 0000000..cc59e31 --- /dev/null +++ b/tests/test_set_hostname.py @@ -0,0 +1,72 @@ +"""set_hostname: what it writes to /etc/hosts (napalm-proxmox#21).""" + +from __future__ import annotations + +import base64 +import re + +import pytest + +from napalm_proxmox import ProxmoxDriver + +HOSTS = """\ +127.0.0.1 localhost.localdomain localhost +# pve is the node; pve-backup its backup target +10.0.0.1 pve.old.lan pve +10.0.0.2 pve-backup.old.lan pve-backup +""" + + +class _Node: + """The node's shell: /etc/hostname reads "pve", /etc/hosts reads HOSTS.""" + + def __init__(self) -> None: + self.ran: list[str] = [] + + def __call__(self, command: str) -> str: + self.ran.append(command) + if command.startswith("cat /etc/hostname"): + return "pve\n" + if command.startswith("cat /etc/hosts"): + return HOSTS + return "" + + def written_hosts(self) -> str: + write = next(c for c in self.ran if c.rstrip().endswith("chmod 644 /etc/hosts")) + return base64.b64decode(re.match(r"echo (\S+) \| base64 -d", write).group(1)).decode() + + +@pytest.fixture +def node() -> _Node: + return _Node() + + +@pytest.fixture +def driver(node: _Node) -> ProxmoxDriver: + d = ProxmoxDriver("pve", "root@pam", "pw") + d._exec_ssh_command = node + return d + + +def test_an_fqdn_renames_the_nodes_names_and_nothing_else(driver, node): + driver.set_hostname("new.dom.lan") + + assert node.written_hosts() == ( + "127.0.0.1 localhost.localdomain localhost\n" + "# pve is the node; pve-backup its backup target\n" + "10.0.0.1 new.dom.lan new\n" + "10.0.0.2 pve-backup.old.lan pve-backup\n" + ) + + +def test_a_short_name_keeps_the_old_domain(driver, node): + driver.set_hostname("new") + + assert "10.0.0.1 new.old.lan new\n" in node.written_hosts() + assert "10.0.0.2 pve-backup.old.lan pve-backup\n" in node.written_hosts() + + +def test_the_same_name_leaves_hosts_alone(driver, node): + driver.set_hostname("pve") + + assert not any("/etc/hosts" in c and "base64 -d" in c for c in node.ran) From 037a725b746c04730a410a81c808010605bc0e72 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:48:48 +0100 Subject: [PATCH 06/14] fix(updates): report a failed apply_updates as a failure apt-get ran with "|| true" and the result was always success, so a failed upgrade read as done. The exit status is read now and a failure names it. Closes #20 --- CHANGELOG.md | 2 ++ napalm_proxmox/system_mixin.py | 14 ++++++++++---- tests/test_updates.py | 29 +++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd9e856..b40655f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `set_hostname()` renames only the node's own names in `/etc/hosts`: other hosts whose names start with the old one (`pve-backup`) and comments stay as they are, and the old FQDN becomes the new one instead of a mix of both (#21). +- `apply_updates()` reports a failed upgrade as `success: False`, with apt-get's + exit status in `error`; it reported success whatever apt-get did (#20). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index 7f63931..53ea748 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -474,11 +474,17 @@ class ProxmoxSystemMixin: if not re.match(r"^[a-zA-Z0-9_\-\+\.]+$", pkg): raise ValueError(f"Invalid package name: {pkg!r}") pkg_args = " ".join(packages) - output = self._exec_ssh_command( - "DEBIAN_FRONTEND=noninteractive apt-get install --only-upgrade -y " - f"{pkg_args} 2>&1 || true" + output, status = self._rc( + f"DEBIAN_FRONTEND=noninteractive apt-get install --only-upgrade -y {pkg_args}" ) - return {"success": True, "output": output} + if status == 0: + return {"success": True, "output": output} + error = ( + f"apt-get exited with status {status}" + if status >= 0 + else "apt-get's exit status could not be read" + ) + return {"success": False, "output": output, "error": error} # ------------------------------------------------------------------ # # SNMP / Health diff --git a/tests/test_updates.py b/tests/test_updates.py index 1ca67d5..41cac64 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -104,6 +104,35 @@ class TestRefresh: assert result == {"success": False, "output": "403 Permission check failed"} +class TestApplyUpdates: + def test_a_clean_upgrade_succeeds(self, driver): + with patch.object(driver, "_exec_ssh_command", return_value="Done.\n__PVE_RC=0") as exec_: + result = driver.apply_updates(["curl"]) + + assert "apt-get install --only-upgrade -y curl" in exec_.call_args[0][0] + assert result == {"success": True, "output": "Done."} + + def test_a_failed_upgrade_is_a_failure(self, driver): + out = "E: Could not get lock /var/lib/dpkg/lock-frontend\n__PVE_RC=100" + with patch.object(driver, "_exec_ssh_command", return_value=out): + result = driver.apply_updates(["curl"]) + + assert result["success"] is False + assert result["output"] == "E: Could not get lock /var/lib/dpkg/lock-frontend" + assert result["error"] == "apt-get exited with status 100" + + def test_an_unread_exit_status_is_no_success(self, driver): + with patch.object(driver, "_exec_ssh_command", return_value=""): + result = driver.apply_updates(["curl"]) + + assert result["success"] is False + assert result["error"] == "apt-get's exit status could not be read" + + def test_a_name_that_is_no_package_name_is_refused(self, driver): + with pytest.raises(ValueError): + driver.apply_updates(["curl;reboot"]) + + class TestHostStatus: def test_the_node_is_read_over_the_exec_path(self, driver): report = ( From 4ee74e4b3d904e0a549dee31854e351dfdfac183 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:50:16 +0100 Subject: [PATCH 07/14] fix(users): list real login accounts and read admins from the ACL The passwd filter's precedence was inverted: it listed uid < 1000 (daemon, bin, ...) and skipped regular users. The level check looked for "Administrator" in the user record, which never names roles. The level now comes from /access/acl (Administrator on /, by user or group), root@pam always 15. Closes #19 --- CHANGELOG.md | 5 ++ napalm_proxmox/system_mixin.py | 111 +++++++++++++++++++++------------ tests/quality_baseline.json | 2 - tests/test_users.py | 66 ++++++++++++++++++++ 4 files changed, 142 insertions(+), 42 deletions(-) create mode 100644 tests/test_users.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b40655f..ca4f96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 they are, and the old FQDN becomes the new one instead of a mix of both (#21). - `apply_updates()` reports a failed upgrade as `success: False`, with apt-get's exit status in `error`; it reported success whatever apt-get did (#20). +- `get_users()` lists the node's root and regular login accounts (uid 1000 and + up) instead of its system accounts (`daemon`, `bin`, …), and gives a Proxmox + user level 15 when the ACL grants it the Administrator role on `/`, directly + or through a group; `root@pam` is always 15. Before, practically every + Proxmox user was level 1 (#19). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index 53ea748..a1903ba 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -73,6 +73,40 @@ def _rename_in_hosts(hosts: str, old_short: str, new_short: str, new_domain: str return "".join(lines) +#: Proxmox's superuser, whatever the ACL says. +_PVE_SUPERUSER = "root@pam" +#: The role that makes a Proxmox user an administrator of the node. +_PVE_ADMIN_ROLE = "Administrator" +#: Regular login accounts: Debian gives them uids from 1000 up; 65534 is nobody. +_FIRST_REGULAR_UID = 1000 +_NOBODY_UID = 65534 + + +def _words(value: Any, sep: str | None = None) -> list[str]: + return [w.strip() for w in str(value or "").split(sep) if w.strip()] + + +def _user_entry(level: int) -> _JsonDict: + # NAPALM's get_users shape; the password is never read. + return {"level": level, "password": "", "sshkeys": []} # nosec B105 + + +def _login_accounts(passwd: str) -> list[tuple[str, int]]: + """``(name, uid)`` of root and every regular account in *passwd*.""" + accounts = [] + for line in passwd.splitlines(): + parts = line.split(":") + if len(parts) < 7: + continue + try: + uid = int(parts[2]) + except ValueError: + continue + if uid == 0 or _FIRST_REGULAR_UID <= uid < _NOBODY_UID: + accounts.append((parts[0], uid)) + return accounts + + def _net(value: str) -> str: return str(ipaddress.ip_network(value.strip(), strict=False)) @@ -239,55 +273,52 @@ class ProxmoxSystemMixin: def get_users(self) -> dict[str, _JsonDict]: """Return users configured on the Proxmox node. - Reads from both the Proxmox access/users API and local /etc/passwd. + Proxmox's own users (``/access/users``) at level 15 when they hold the + Administrator role on the whole datacenter (``/``), directly or through + a group, 1 otherwise; ``root@pam`` is always 15. Then the node's login + accounts from the passwd database: root (15) and regular users + (uid 1000 and up, without ``nobody``) at 0; system accounts are left out. """ result: dict[str, _JsonDict] = {} - try: - pve_users = self._api.access.users.get() or [] # type: ignore[union-attr] - except Exception as exc: - logger.debug("Failed to fetch Proxmox users: %s", exc) - pve_users = [] - - for user in pve_users: + admins, admin_groups = self._pve_administrators() + for user in self._pve_users(): uid = user.get("userid", "") if not uid: continue - # Proxmox roles: Administrator -> 15, otherwise 1 - level = 1 - try: - roles = self._api.access.users(uid).get() or {} # type: ignore[union-attr] - if "Administrator" in str(roles): - level = 15 - except Exception as exc: - logger.debug("Failed to fetch roles for user %s: %s", uid, exc) - result[uid] = { - "level": level, - # NAPALM's get_users shape; the password is never read. - "password": "", # nosec B105 - "sshkeys": [], - } + groups = set(_words(user.get("groups"), sep=",")) + is_admin = uid == _PVE_SUPERUSER or uid in admins or bool(groups & admin_groups) + result[uid] = _user_entry(15 if is_admin else 1) - # Merge local OS users from /etc/passwd raw = self._exec_ssh_command("getent passwd 2>/dev/null || cat /etc/passwd") - for line in raw.splitlines(): - parts = line.split(":") - if len(parts) < 7: - continue - uname, _, uid_str, *_ = parts - try: - uid_int = int(uid_str) - except ValueError: - continue - if uname not in result and uid_int < 1000 or uid_int == 0: - result[uname] = { - "level": 15 if uid_int == 0 else 0, - # NAPALM's get_users shape; the password is never read. - "password": "", # nosec B105 - "sshkeys": [], - } - + for uname, uid_int in _login_accounts(raw): + result.setdefault(uname, _user_entry(15 if uid_int == 0 else 0)) return result + def _pve_users(self) -> list[_JsonDict]: + try: + return self._api.access.users.get() or [] # type: ignore[union-attr] + except Exception as exc: + logger.debug("Failed to fetch Proxmox users: %s", exc) + return [] + + def _pve_administrators(self) -> tuple[set[str], set[str]]: + """Users and groups holding the Administrator role on ``/`` (``/access/acl``).""" + try: + acl = self._api.access.acl.get() or [] # type: ignore[union-attr] + except Exception as exc: + logger.debug("Failed to fetch the Proxmox ACL: %s", exc) + acl = [] + users: set[str] = set() + groups: set[str] = set() + for entry in acl: + if entry.get("path") != "/" or entry.get("roleid") != _PVE_ADMIN_ROLE: + continue + if entry.get("type") == "user": + users.add(entry.get("ugid", "")) + elif entry.get("type") == "group": + groups.add(entry.get("ugid", "")) + return users, groups + # ------------------------------------------------------------------ # # Kernel facts (KernelFactsMixin supplies get_kernel_facts) # ------------------------------------------------------------------ # diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 17d19e2..ef77295 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -16,7 +16,6 @@ "napalm_proxmox/sdn_mixin.py::ProxmoxSDNMixin.set_vlan": 13, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_disk_smart": 22, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_packages": 16, - "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_users": 15, "napalm_proxmox/utils.py::addresses_from_node_network": 15, "napalm_proxmox/vm_contract_mixin.py::parse_vm_config": 16, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin._get_vm_disk_and_boot": 16, @@ -35,7 +34,6 @@ "napalm_proxmox/sdn_mixin.py::ProxmoxSDNMixin.set_vlan": 70, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_disk_smart": 74, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_packages": 66, - "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.get_users": 51, "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.set_hostname": 114, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vm_interfaces": 179, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vms": 114, diff --git a/tests/test_users.py b/tests/test_users.py new file mode 100644 index 0000000..c178fd7 --- /dev/null +++ b/tests/test_users.py @@ -0,0 +1,66 @@ +"""get_users: Proxmox users with their level, and the node's login accounts (#19).""" + +from __future__ import annotations + +from unittest.mock import patch + +PASSWD = """\ +root:x:0:0:root:/root:/bin/bash +daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin +bin:x:2:2:bin:/bin:/usr/sbin/nologin +www-data:x:33:33:www-data:/var/www:/usr/sbin/nologin +alice:x:1000:1000:Alice:/home/alice:/bin/bash +nobody:x:65534:65534:nobody:/nonexistent:/usr/sbin/nologin +""" + +# GET /access/acl: who holds which role where. +ACL = [ + {"path": "/", "roleid": "Administrator", "type": "user", "ugid": "admin@pve", "propagate": 1}, + {"path": "/", "roleid": "Administrator", "type": "group", "ugid": "ops", "propagate": 1}, + {"path": "/vms/100", "roleid": "Administrator", "type": "user", "ugid": "vm@pve"}, + {"path": "/", "roleid": "PVEAuditor", "type": "user", "ugid": "audit@pve", "propagate": 1}, +] + +USERS = [ + {"userid": "root@pam", "groups": ""}, + {"userid": "admin@pve", "groups": ""}, + {"userid": "bob@pve", "groups": "ops,dev"}, + {"userid": "vm@pve", "groups": ""}, + {"userid": "audit@pve", "groups": ""}, +] + + +def _users(driver, passwd=PASSWD): + driver._api.access.users.get.return_value = USERS + driver._api.access.acl.get.return_value = ACL + with patch.object(driver, "_exec_ssh_command", return_value=passwd): + return driver.get_users() + + +class TestLocalAccounts: + def test_root_and_real_users_not_system_accounts(self, driver): + result = _users(driver) + + assert result["root"]["level"] == 15 + assert "alice" in result + for system in ("daemon", "bin", "www-data", "nobody"): + assert system not in result + + +class TestProxmoxUsers: + def test_administrator_on_the_whole_datacenter_is_level_15(self, driver): + result = _users(driver) + + assert result["admin@pve"]["level"] == 15 + + def test_administrator_through_a_group_is_level_15(self, driver): + assert _users(driver)["bob@pve"]["level"] == 15 + + def test_root_at_pam_is_always_level_15(self, driver): + assert _users(driver)["root@pam"]["level"] == 15 + + def test_another_role_or_a_narrower_path_is_level_1(self, driver): + result = _users(driver) + + assert result["audit@pve"]["level"] == 1 + assert result["vm@pve"]["level"] == 1 From 5195cebfcabd0a4b8ee7a1f07b997e690bd5bb61 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:52:03 +0100 Subject: [PATCH 08/14] fix(ntp): read chrony's sources in get_ntp_stats The getter ran "chronyc tracking || ntpq -pn" and parsed ntpq's format. Proxmox VE ships chrony, so ntpq never ran and nothing matched. chrony is now read in CSV mode (sources plus sourcestats); ntpq stays the fallback and keeps peers without a tally mark. Closes #18 --- CHANGELOG.md | 4 ++ napalm_proxmox/ntp.py | 113 +++++++++++++++++++++++++++++++++ napalm_proxmox/system_mixin.py | 34 ++-------- tests/test_ntp_stats.py | 86 +++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 30 deletions(-) create mode 100644 napalm_proxmox/ntp.py create mode 100644 tests/test_ntp_stats.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4f96b..e3845bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 user level 15 when the ACL grants it the Administrator role on `/`, directly or through a group; `root@pam` is always 15. Before, practically every Proxmox user was level 1 (#19). +- `get_ntp_stats()` reports chrony's sources, Proxmox VE's default, instead of + always returning an empty list: offset in ms, the poll interval in seconds, + the offset's standard deviation as `jitter`; chrony reports no delay, so + `delay` is 0.0. On ntpd, a peer without tally mark is no longer dropped (#18). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/ntp.py b/napalm_proxmox/ntp.py new file mode 100644 index 0000000..c795dd8 --- /dev/null +++ b/napalm_proxmox/ntp.py @@ -0,0 +1,113 @@ +"""NTP statistics (``get_ntp_stats``) from chrony or ntpd, in NAPALM's NTPStats shape. + +Proxmox VE ships chrony; ntpd is read only where chrony is not installed. +chrony is read in its CSV mode (``chronyc -c``): ``sources`` names each source +with its state, ``sourcestats`` adds the spread of its offset, reported as +jitter. chrony reports no delay per source, so ``delay`` is 0.0. Times are in +milliseconds and the poll interval in seconds, as ``ntpq -pn`` gives them. +""" + +from __future__ import annotations + +import re +from typing import Any + +CHRONY_SOURCES_MARK = "CHRONY_SOURCES" +CHRONY_SOURCESTATS_MARK = "CHRONY_SOURCESTATS" + +#: chrony when it is installed, ntpd's ntpq otherwise. +NTP_STATS_COMMAND = ( + "if command -v chronyc >/dev/null 2>&1; then " + f"echo {CHRONY_SOURCES_MARK}; chronyc -n -c sources; " + f"echo {CHRONY_SOURCESTATS_MARK}; chronyc -n -c sourcestats; " + "else ntpq -pn; fi 2>/dev/null" +) + +#: chrony's source mode as ntpq's type: server (unicast), peer (symmetric), refclock (local). +_CHRONY_TYPES = {"^": "u", "=": "s", "#": "l"} +_MS_PER_S = 1000.0 + +#: ``*192.168.1.1 10.0.0.1 4 u 12 64 377 1.234 0.567 0.123``; the first +#: column is the tally mark, a space for a peer without one. +_NTPQ_LINE = re.compile( + r"^([ *+#ox.-])(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\d+)\s+(\d+)" + r"\s+([-\d.]+)\s+([-\d.]+)\s+([-\d.]+)" +) + + +def parse_ntp_stats(output: str) -> list[dict[str, Any]]: + """The NTPStats list in the output of :data:`NTP_STATS_COMMAND`.""" + if output.lstrip().startswith(CHRONY_SOURCES_MARK): + return parse_chrony(output) + return parse_ntpq(output) + + +def parse_chrony(output: str) -> list[dict[str, Any]]: + """Sources and their statistics from ``chronyc -n -c sources`` / ``sourcestats``.""" + sources, _, sourcestats = output.partition(CHRONY_SOURCESTATS_MARK) + jitter = _chrony_jitter(sourcestats) + stats = [] + for line in sources.splitlines(): + fields = line.strip().split(",") + if len(fields) < 10: + continue + mode, state, name, stratum, poll, reach, last_rx, _, offset, _ = fields[:10] + try: + stats.append( + { + "remote": name, + "referenceid": "", + "synchronized": state == "*", + "stratum": int(stratum), + "type": _CHRONY_TYPES.get(mode, ""), + "when": last_rx, + "hostpoll": 2 ** int(poll), + "reachability": int(reach), + "delay": 0.0, + "offset": float(offset) * _MS_PER_S, + "jitter": jitter.get(name, 0.0), + } + ) + except ValueError: + continue + return stats + + +def _chrony_jitter(sourcestats: str) -> dict[str, float]: + """Each source's offset standard deviation in ms (``sourcestats``' last column).""" + jitter = {} + for line in sourcestats.splitlines(): + fields = line.strip().split(",") + if len(fields) < 8: + continue + try: + jitter[fields[0]] = float(fields[7]) * _MS_PER_S + except ValueError: + continue + return jitter + + +def parse_ntpq(output: str) -> list[dict[str, Any]]: + """Peers from ``ntpq -pn``.""" + stats = [] + for line in output.splitlines(): + m = _NTPQ_LINE.match(line) + if not m: + continue + tally, remote, refid, stratum, kind, when, poll, reach, delay, offset, jitter = m.groups() + stats.append( + { + "remote": remote, + "referenceid": refid, + "synchronized": tally == "*", + "stratum": int(stratum), + "type": kind, + "when": when, + "hostpoll": int(poll), + "reachability": int(reach), + "delay": float(delay), + "offset": float(offset), + "jitter": float(jitter), + } + ) + return stats diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index a1903ba..669f078 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -28,6 +28,8 @@ from typing import Any from napalm_device_types import APT_UPGRADABLE_COMMAND, parse_apt_upgradable +from napalm_proxmox.ntp import NTP_STATS_COMMAND, parse_ntp_stats + logger = logging.getLogger(__name__) _JsonDict = dict[str, Any] @@ -197,36 +199,8 @@ class ProxmoxSystemMixin: return servers def get_ntp_stats(self) -> list[_JsonDict]: - """Return NTP synchronisation statistics from chronyc/ntpq output.""" - raw = self._exec_ssh_command( - "chronyc -n tracking 2>/dev/null || ntpq -pn 2>/dev/null || true" - ) - stats: list[_JsonDict] = [] - for line in raw.splitlines(): - line = line.strip() - # ntpq -pn format: *remote refid st t when poll reach delay offset jitter - m = re.match( - r"^([\*\+\-\s])([\d.]+)\s+([\d.]+)\s+(\d+)\s+\S+\s+(\S+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+([-\d.]+)\s+([\d.]+)", - line, - ) - if m: - synced = m.group(1).strip() == "*" - stats.append( - { - "remote": m.group(2), - "referenceid": m.group(3), - "synchronized": synced, - "stratum": int(m.group(4)), - "type": "", - "when": m.group(5), - "hostpoll": int(m.group(6)), - "reachability": int(m.group(7)), - "delay": float(m.group(8)), - "offset": float(m.group(9)), - "jitter": float(m.group(10)), - } - ) - return stats + """Return NTP synchronisation statistics from chrony, or ntpd where chrony is missing.""" + return parse_ntp_stats(self._exec_ssh_command(NTP_STATS_COMMAND)) # ------------------------------------------------------------------ # # get_snmp_information diff --git a/tests/test_ntp_stats.py b/tests/test_ntp_stats.py new file mode 100644 index 0000000..6f41a28 --- /dev/null +++ b/tests/test_ntp_stats.py @@ -0,0 +1,86 @@ +"""get_ntp_stats on chrony (Proxmox VE's default) and on ntpd (#18).""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +# chrony 4.x, ``chronyc -n -c sources`` then ``chronyc -n -c sourcestats``, +# behind the markers the driver's command prints. +CHRONY = """\ +CHRONY_SOURCES +^,*,192.168.1.1,2,6,377,8,-0.000012345,-0.000013000,0.012345678 +^,+,10.0.0.2,3,10,17,301,0.000250000,0.000251000,0.020000000 +^,?,10.0.0.3,0,6,0,-,0.000000000,0.000000000,0.000000000 +CHRONY_SOURCESTATS +192.168.1.1,10,6,320,-0.012,0.150,-0.000001234,0.000045678 +10.0.0.2,6,4,1200,0.003,0.400,0.000200000,0.000500000 +""" + +# ntpd's ``ntpq -pn``: a candidate without tally mark keeps its leading space. +NTPQ = """\ + remote refid st t when poll reach delay offset jitter +============================================================================== +*192.168.1.1 10.0.0.1 4 u 12 64 377 1.234 0.567 0.123 + 192.168.1.5 10.0.0.1 3 u 40 64 377 2.000 0.100 0.050 +""" + + +def _stats(driver, output): + with patch.object(driver, "_exec_ssh_command", return_value=output): + return driver.get_ntp_stats() + + +class TestChrony: + def test_every_source_is_reported(self, driver): + assert [s["remote"] for s in _stats(driver, CHRONY)] == [ + "192.168.1.1", + "10.0.0.2", + "10.0.0.3", + ] + + def test_the_selected_source_is_synchronized(self, driver): + stats = _stats(driver, CHRONY) + + assert [s["synchronized"] for s in stats] == [True, False, False] + + def test_fields_in_ntpq_units(self, driver): + first = _stats(driver, CHRONY)[0] + + assert first == { + "remote": "192.168.1.1", + "referenceid": "", + "synchronized": True, + "stratum": 2, + "type": "u", + "when": "8", + "hostpoll": 64, + "reachability": 377, + "delay": 0.0, + "offset": pytest.approx(-0.013), + "jitter": pytest.approx(0.045678), + } + + def test_a_source_without_statistics_has_no_jitter(self, driver): + last = _stats(driver, CHRONY)[2] + + assert last["jitter"] == 0.0 + assert last["when"] == "-" + assert last["reachability"] == 0 + + def test_the_command_asks_chrony_first(self, driver): + with patch.object(driver, "_exec_ssh_command", return_value="") as exec_: + driver.get_ntp_stats() + + assert "chronyc -n -c sources" in exec_.call_args[0][0] + assert "ntpq -pn" in exec_.call_args[0][0] + + +class TestNtpd: + def test_a_peer_without_tally_mark_is_kept(self, driver): + stats = _stats(driver, NTPQ) + + assert [s["remote"] for s in stats] == ["192.168.1.1", "192.168.1.5"] + assert [s["synchronized"] for s in stats] == [True, False] + assert stats[1]["stratum"] == 3 From f491f9f9ec7d9b79763dc9794b146327696dd026 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:53:47 +0100 Subject: [PATCH 09/14] fix(ssh): log in with any key type, the key before the password Only RSA keys were read, and a key was used only when no password was set; with the API password always present it never was. The key is now read as Ed25519, ECDSA or RSA and offered first, the password second. The rest of the issue (the API exec branch, errors read as empty output) belongs to the shared transport of NAPALM/napalm-device-types#35. Refs #17 --- CHANGELOG.md | 4 ++ napalm_proxmox/driver.py | 29 ++++++++++---- tests/test_ssh_login_key.py | 79 +++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 tests/test_ssh_login_key.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e3845bf..690e7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 always returning an empty list: offset in ms, the poll interval in seconds, the offset's standard deviation as `jitter`; chrony reports no delay, so `delay` is 0.0. On ntpd, a peer without tally mark is no longer dropped (#18). +- The SSH login to the node takes an Ed25519 or ECDSA key in + `ssh_private_key_str`, not only RSA, and uses the key even when a password is + set too (the password stays the fallback). A key that cannot be read is + logged as a warning and the password used (#17). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/driver.py b/napalm_proxmox/driver.py index 8ca45dc..25305a3 100644 --- a/napalm_proxmox/driver.py +++ b/napalm_proxmox/driver.py @@ -343,17 +343,17 @@ class ProxmoxDriver( import paramiko if self._ssh_client is None: - ssh_pass = self._ssh_password or self.password + # A key is tried first; paramiko falls back to the password. connect_kwargs: _JsonDict = { "username": self._ssh_username or self.username, "timeout": self.timeout, } - if self._ssh_key and not ssh_pass: - from io import StringIO as _StringIO - - connect_kwargs["pkey"] = paramiko.RSAKey.from_private_key(_StringIO(self._ssh_key)) - else: - connect_kwargs["password"] = ssh_pass + password = self._ssh_password or self.password + if password: + connect_kwargs["password"] = password + pkey = _load_private_key(self._ssh_key) if self._ssh_key else None + if pkey is not None: + connect_kwargs["pkey"] = pkey client = paramiko.SSHClient() try: HostKeyCheck(self.optional_args, self.hostname, 22).connect( @@ -511,3 +511,18 @@ class ProxmoxDriver( def cli(self, commands: list[str], encoding: str = "text") -> dict[str, str]: """Execute a list of shell commands and return their outputs.""" return {cmd: self._exec_ssh_command(cmd) for cmd in commands} + + +def _load_private_key(text: str) -> Any: # paramiko.PKey; paramiko is imported lazily + """The SSH private key in *text*, whatever its type, or None when it cannot be read.""" + import io + + import paramiko + + for key_class in (paramiko.Ed25519Key, paramiko.ECDSAKey, paramiko.RSAKey): + try: + return key_class.from_private_key(io.StringIO(text)) + except (paramiko.SSHException, ValueError): + continue + logger.warning("The SSH private key could not be read; logging in with the password") + return None diff --git a/tests/test_ssh_login_key.py b/tests/test_ssh_login_key.py new file mode 100644 index 0000000..d9397d4 --- /dev/null +++ b/tests/test_ssh_login_key.py @@ -0,0 +1,79 @@ +"""The node's SSH login takes any key type, and a key wins over the API password (#17).""" + +from __future__ import annotations + +import io + +import paramiko +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from napalm_proxmox.driver import ProxmoxDriver +from tests.test_ssh_host_key import client # noqa: F401 - the fake paramiko client fixture + + +def _ed25519_pem() -> str: + key = ed25519.Ed25519PrivateKey.generate() + return key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.OpenSSH, + serialization.NoEncryption(), + ).decode() + + +def _paramiko_pem(key: paramiko.PKey) -> str: + out = io.StringIO() + key.write_private_key(out) + return out.getvalue() + + +def _driver(key: str, ssh_password: str | None = None) -> ProxmoxDriver: + args = {"ssh_private_key_str": key} + if ssh_password is not None: + args["ssh_password"] = ssh_password + return ProxmoxDriver("10.0.0.9", "root", "api-s3cret", optional_args=args) + + +@pytest.mark.parametrize( + ("pem", "kind"), + [ + (_ed25519_pem(), paramiko.Ed25519Key), + (_paramiko_pem(paramiko.ECDSAKey.generate()), paramiko.ECDSAKey), + (_paramiko_pem(paramiko.RSAKey.generate(1024)), paramiko.RSAKey), + ], +) +def test_any_key_type_logs_in_although_an_api_password_is_set(client, pem, kind): # noqa: F811 + _driver(pem)._node_ssh() + + assert isinstance(client.connected_with["pkey"], kind) + + +def test_the_password_stays_as_the_fallback(client): # noqa: F811 + _driver(_ed25519_pem(), ssh_password="ssh-pw")._node_ssh() + + assert client.connected_with["password"] == "ssh-pw" + + +def test_without_key_the_password_logs_in(client): # noqa: F811 + ProxmoxDriver("10.0.0.9", "root", "api-s3cret")._node_ssh() + + assert client.connected_with["password"] == "api-s3cret" + assert "pkey" not in client.connected_with + + +def test_a_key_that_cannot_be_read_leaves_the_password(client, caplog): # noqa: F811 + _driver("not a key")._node_ssh() + + assert "pkey" not in client.connected_with + assert client.connected_with["password"] == "api-s3cret" + assert "not a key" not in caplog.text + assert "could not be read" in caplog.text + + +def test_a_token_only_login_sends_no_empty_password(client): # noqa: F811 + ProxmoxDriver( + "10.0.0.9", "root", "", optional_args={"ssh_private_key_str": _ed25519_pem()} + )._node_ssh() + + assert "password" not in client.connected_with From 29ed59c8d1b8a4a5c47874577f05c500aff9b30e Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:54:48 +0100 Subject: [PATCH 10/14] fix(vm-provision): wait for destroy_vm's delete task DELETE /nodes/{node}/qemu/{vmid} starts a task and returns its UPID; destroy_vm ignored it and returned while the VM could still exist, and a failed delete went unnoticed. The task is now awaited. Reading the snippets before the delete, the issue's first point, was fixed in ecff2b4. Closes #15 --- CHANGELOG.md | 2 ++ napalm_proxmox/vm_provision_mixin.py | 26 ++++++++++++-------- tests/quality_baseline.json | 2 +- tests/test_vm_provision_mixin.py | 36 ++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 690e7b0..908c05c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `ssh_private_key_str`, not only RSA, and uses the key even when a password is set too (the password stays the fallback). A key that cannot be read is logged as a warning and the password used (#17). +- `destroy_vm()` returns once Proxmox has deleted the VM, not as soon as the + delete was queued, and raises when the delete fails (#15). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/vm_provision_mixin.py b/napalm_proxmox/vm_provision_mixin.py index 0d7577d..bce78f5 100644 --- a/napalm_proxmox/vm_provision_mixin.py +++ b/napalm_proxmox/vm_provision_mixin.py @@ -819,16 +819,7 @@ class ProxmoxVMProvisionMixin: snippets = self._cicustom_snippets(vmid_int) # Step 2: Delete VM - # Proxmox's API parameter is hyphenated (destroy-unreferenced-disks), - # not a valid Python identifier — proxmoxer forwards kwargs to the - # request verbatim with no underscore-to-hyphen translation, so this - # must be built as a dict and unpacked rather than passed as a kwarg. - _logger.debug(f"Deleting VM {vmid} configuration and disks") - delete_params = { - "purge": 1, - "destroy-unreferenced-disks": 1 if remove_disk else 0, - } - self._node_api().qemu(vmid_int).delete(**delete_params) + self._delete_vm(vmid_int, remove_disk, timeout) # Step 3: Clean up Cloud-Init snippets # (This is best-effort; snippet files may be unreachable if storage is unavailable) @@ -845,6 +836,21 @@ class ProxmoxVMProvisionMixin: _logger.exception(f"Failed to destroy VM {vmid}: {e}") raise + def _delete_vm(self, vmid: int, remove_disk: bool, timeout: int) -> None: + """Delete the VM and wait for it to be gone: the delete is a task.""" + _logger.debug(f"Deleting VM {vmid} configuration and disks") + # Proxmox's API parameter is hyphenated (destroy-unreferenced-disks), + # not a valid Python identifier — proxmoxer forwards kwargs to the + # request verbatim with no underscore-to-hyphen translation, so this + # must be built as a dict and unpacked rather than passed as a kwarg. + delete_params = { + "purge": 1, + "destroy-unreferenced-disks": 1 if remove_disk else 0, + } + upid = self._node_api().qemu(vmid).delete(**delete_params) + if upid: + self._wait_for_task(upid, timeout=timeout) + def _cicustom_snippets(self, vmid: int) -> list[tuple[str, str]]: """``(storage, path)`` of every snippet the VM's cicustom names. diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index ef77295..fdceda3 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -39,7 +39,7 @@ "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vms": 114, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin._download_cloud_image": 61, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.create_vm_from_cloud_init": 232, - "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.destroy_vm": 65, + "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.destroy_vm": 56, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.get_vm_status": 92 }, "mypy_relaxed": [ diff --git a/tests/test_vm_provision_mixin.py b/tests/test_vm_provision_mixin.py index 06fbe36..e828903 100644 --- a/tests/test_vm_provision_mixin.py +++ b/tests/test_vm_provision_mixin.py @@ -484,6 +484,42 @@ def test_destroy_vm_already_stopped(): assert mock_vm.delete.called +def _destroy_mixin(delete_task: dict) -> tuple[ProxmoxVMProvisionMixin, MagicMock]: + mixin = ProxmoxVMProvisionMixin() + node = MagicMock() + mixin._node_api = MagicMock(return_value=node) + vm = node.qemu.return_value + vm.status.stop.post.return_value = "UPID:pve1:125:stop" + vm.config.get.return_value = {} + vm.delete.return_value = "UPID:pve1:126:qmdestroy" + tasks = { + "UPID:pve1:125:stop": {"status": "stopped", "exitstatus": "OK"}, + "UPID:pve1:126:qmdestroy": delete_task, + } + node.tasks.side_effect = lambda upid: MagicMock(**{"status.get.return_value": tasks[upid]}) + return mixin, node + + +def test_destroy_vm_waits_for_the_delete_task(): + """destroy_vm returns once the VM is gone, not when its delete was queued (#15).""" + mixin, node = _destroy_mixin({"status": "stopped", "exitstatus": "OK"}) + + with patch("time.sleep"): + mixin.destroy_vm("101") + + assert [c.args[0] for c in node.tasks.call_args_list] == [ + "UPID:pve1:125:stop", + "UPID:pve1:126:qmdestroy", + ] + + +def test_destroy_vm_raises_when_the_delete_task_fails(): + mixin, _ = _destroy_mixin({"status": "stopped", "exitstatus": "VM is locked (backup)"}) + + with patch("time.sleep"), pytest.raises(RuntimeError, match="VM is locked"): + mixin.destroy_vm("101") + + # --------------------------------------------------------------------------- # get_network_targets # --------------------------------------------------------------------------- From 10cf788b86f64d303b33a579f5d65f3dc1423678 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:55:32 +0100 Subject: [PATCH 11/14] fix(ping): parse each reply into results The probe pattern expected "from" after icmp_seq, but iputils prints "bytes from : icmp_seq=... time=... ms", so results was always empty. Replies by name ("from host (ip)") and over IPv6 are read too. Closes #12 --- CHANGELOG.md | 2 ++ napalm_proxmox/routing_mixin.py | 10 +++++++--- tests/test_misc.py | 25 +++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908c05c..1bb4fd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 logged as a warning and the password used (#17). - `destroy_vm()` returns once Proxmox has deleted the VM, not as soon as the delete was queued, and raises when the delete fails (#15). +- `ping()` fills `results` with each reply's address and round-trip time; it + was always empty (#12). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/routing_mixin.py b/napalm_proxmox/routing_mixin.py index edf474e..df3ede7 100644 --- a/napalm_proxmox/routing_mixin.py +++ b/napalm_proxmox/routing_mixin.py @@ -23,6 +23,10 @@ import napalm.base.constants as C _JsonDict = dict[str, Any] +#: One reply in iputils' ping output, by address or by name with the address +#: in brackets: "64 bytes from dns.google (8.8.4.4): icmp_seq=1 ttl=118 time=10.5 ms". +_PING_REPLY = re.compile(r"bytes from (?:\S+ \()?([0-9a-fA-F.:]+)\)?: .*?time=([\d.]+) ms") + class ProxmoxRoutingMixin: """Mixin providing routing NAPALM methods (get_route_to, ping, traceroute).""" @@ -90,9 +94,9 @@ class ProxmoxRoutingMixin: rtt_std = float(m_rtt.group(4)) if m_rtt else 0.0 # Individual probe lines - probes = [] - for m_probe in re.finditer(r"icmp_seq=\d+.*?time=([\d.]+) ms.*?from ([\d.a-fA-F:]+)", raw): - probes.append({"ip_address": m_probe.group(2), "rtt": float(m_probe.group(1))}) + probes = [ + {"ip_address": m.group(1), "rtt": float(m.group(2))} for m in _PING_REPLY.finditer(raw) + ] return { "success": { diff --git a/tests/test_misc.py b/tests/test_misc.py index 42a660e..c374cac 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -175,6 +175,31 @@ class TestPing: assert result["success"]["packet_loss"] == 0 assert result["success"]["rtt_min"] == pytest.approx(10.5, abs=0.01) + def test_every_reply_is_a_result(self, driver): + """iputils puts "from " before icmp_seq (napalm-proxmox#12).""" + driver._node_api().execute.post.return_value = {"data": self.PING_SUCCESS} + result = driver.ping("8.8.8.8") + assert result["success"]["results"] == [ + {"ip_address": "8.8.8.8", "rtt": 10.5}, + {"ip_address": "8.8.8.8", "rtt": 11.2}, + ] + + def test_a_reply_by_name_and_over_ipv6(self, driver): + output = ( + "PING dns.google (8.8.4.4) 56(84) bytes of data.\n" + "64 bytes from dns.google (8.8.4.4): icmp_seq=1 ttl=118 time=123 ms\n" + "64 bytes from 2001:db8::1: icmp_seq=2 ttl=64 time=0.045 ms\n" + "--- dns.google ping statistics ---\n" + "2 packets transmitted, 2 received, 0% packet loss, time 1001ms\n" + "rtt min/avg/max/mdev = 0.045/61.5/123.0/61.4 ms\n" + ) + driver._node_api().execute.post.return_value = {"data": output} + result = driver.ping("dns.google") + assert result["success"]["results"] == [ + {"ip_address": "8.8.4.4", "rtt": 123.0}, + {"ip_address": "2001:db8::1", "rtt": 0.045}, + ] + def test_failure(self, driver): driver._node_api().execute.post.return_value = {"data": self.PING_FAIL} result = driver.ping("badhost") From 9c7b64ba6456b754225db182ec20be22b6d222d5 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 21:56:29 +0100 Subject: [PATCH 12/14] fix(vm-provision): say why a cloud-image download failed, drop its tmp wget ran with -q, so a failure carried no reason, and a failed download left its partial .tmp in the cache, taking the same space again on every retry. wget now runs with -nv and the .tmp is removed on failure, keeping wget's exit status. Closes #3 --- CHANGELOG.md | 3 ++ napalm_proxmox/vm_provision_mixin.py | 21 ++++++++---- tests/quality_baseline.json | 2 +- tests/test_vm_provision_mixin.py | 49 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bb4fd9..d48777c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 delete was queued, and raises when the delete fails (#15). - `ping()` fills `results` with each reply's address and round-trip time; it was always empty (#12). +- A failed cloud-image download in `create_vm_from_cloud_init()` says why + (wget's own message, such as "No space left on device") and removes its + partial `.tmp` file instead of leaving it to fill the node's disk (#3). ## [0.8.0] - 2026-10-23 diff --git a/napalm_proxmox/vm_provision_mixin.py b/napalm_proxmox/vm_provision_mixin.py index bce78f5..02d6194 100644 --- a/napalm_proxmox/vm_provision_mixin.py +++ b/napalm_proxmox/vm_provision_mixin.py @@ -186,12 +186,7 @@ class ProxmoxVMProvisionMixin: timeout=30, ) if "EXISTS" not in exists: - _logger.info(f"Downloading cloud image {image_url} -> {local_path}") - self._run_node_command( - f"wget -q -O {local_path}.tmp '{image_url}' " - f"&& mv {local_path}.tmp {local_path}", - timeout=timeout, - ) + self._fetch_image(image_url, local_path, timeout) if not image_checksum: return local_path @@ -216,6 +211,20 @@ class ProxmoxVMProvisionMixin: raise AssertionError("unreachable") # loop always returns or raises above + def _fetch_image(self, image_url: str, local_path: str, timeout: int) -> None: + """Download *image_url* to *local_path* on the node, through a ``.tmp``. + + wget runs with ``-nv`` so its reason ("No space left on device") + reaches the error, and a failed download leaves no partial ``.tmp``. + """ + _logger.info(f"Downloading cloud image {image_url} -> {local_path}") + tmp = f"{local_path}.tmp" + self._run_node_command( + f"wget -nv -O {tmp} '{image_url}' && mv {tmp} {local_path} " + f"|| {{ s=$?; rm -f {tmp}; exit $s; }}", + timeout=timeout, + ) + def _cloud_image_on_node(self, image_url: str, image_checksum: str | None, timeout: int) -> str: """The node path of the image, downloaded, verified and unpacked. diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index fdceda3..6c9c8d9 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -37,7 +37,7 @@ "napalm_proxmox/system_mixin.py::ProxmoxSystemMixin.set_hostname": 114, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vm_interfaces": 179, "napalm_proxmox/vm_mixin.py::ProxmoxVMMixin.get_vms": 114, - "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin._download_cloud_image": 61, + "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin._download_cloud_image": 56, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.create_vm_from_cloud_init": 232, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.destroy_vm": 56, "napalm_proxmox/vm_provision_mixin.py::ProxmoxVMProvisionMixin.get_vm_status": 92 diff --git a/tests/test_vm_provision_mixin.py b/tests/test_vm_provision_mixin.py index e828903..d6a0235 100644 --- a/tests/test_vm_provision_mixin.py +++ b/tests/test_vm_provision_mixin.py @@ -1364,3 +1364,52 @@ def test_create_vm_passes_every_ssh_key_one_per_line(): sshkeys = call[1]["sshkeys"] assert unquote(sshkeys) == "\n".join(keys) assert re.fullmatch(r"[-%a-zA-Z0-9_.!~*'()]*", sshkeys) + + +class _LocalShell: + """The node's SSH session as a local ``sh``, *prelude* run before each command.""" + + def __init__(self, prelude: str) -> None: + self.prelude = prelude + + def exec_command(self, command: str, timeout: int | None = None): + import subprocess + + run = subprocess.run( + ["sh", "-c", self.prelude + command], capture_output=True, timeout=timeout + ) + return _mock_ssh_exec(run.returncode, run.stdout.decode(), run.stderr.decode()) + + +# wget as it fails on a full disk: part of the file written, the reason on stderr. +_WGET_DISK_FULL = ( + "wget() { " + 'while [ $# -gt 0 ]; do [ "$1" = -O ] && o=$2; shift; done; ' + 'echo partial > "$o"; ' + 'echo "$o: No space left on device" >&2; return 3; }; ' +) + + +def test_download_cloud_image_failure_says_why_and_leaves_no_tmp(tmp_path, monkeypatch): + """A failed download names wget's reason and removes its .tmp (napalm-proxmox#3).""" + import napalm_proxmox.vm_provision_mixin as module + + monkeypatch.setattr(module, "_IMAGE_CACHE_DIR", str(tmp_path)) + mixin = ProxmoxVMProvisionMixin() + mixin._node_ssh = MagicMock(return_value=_LocalShell(_WGET_DISK_FULL)) + + with pytest.raises(RuntimeError, match="No space left on device") as err: + mixin._download_cloud_image("https://example.org/img/debian.qcow2", None, timeout=10) + + assert "exit 3" in str(err.value) + assert list(tmp_path.iterdir()) == [] + + +def test_download_cloud_image_does_not_silence_wget(): + mixin = ProxmoxVMProvisionMixin() + mixin._run_node_command = MagicMock(side_effect=["MISSING", ""]) + + mixin._download_cloud_image("https://example.org/img/debian.qcow2", None, timeout=10) + + download = mixin._run_node_command.call_args_list[1][0][0] + assert download.startswith("wget -nv ") From 8ef880b73b8781b0e0c371c1604314d5bfa098a1 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:08:54 +0100 Subject: [PATCH 13/14] fix(hostname): refuse to rename a clustered node The rename moves /etc/pve/nodes/ while the node still runs as , which Proxmox does not support for a cluster member. set_hostname now checks for a corosync.conf first and raises before touching anything on a member, or when the check cannot be read; a single node renames as before. The check is the file, not `pvecm status`: pvecm decides membership by /etc/pve/corosync.conf itself, and its status also fails on a member whose corosync is down. /etc/corosync/corosync.conf covers a member whose /etc/pve is not mounted. Closes #21 --- CHANGELOG.md | 5 +++++ napalm_proxmox/system_mixin.py | 30 ++++++++++++++++++++------ tests/test_set_hostname.py | 39 ++++++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d48777c..a0f4b61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `set_hostname()` renames only the node's own names in `/etc/hosts`: other hosts whose names start with the old one (`pve-backup`) and comments stay as they are, and the old FQDN becomes the new one instead of a mix of both (#21). +- `set_hostname()` refuses a node that is part of a cluster (a + `/etc/pve/corosync.conf` or `/etc/corosync/corosync.conf` exists) with a + `CommandErrorException`, before changing anything: it moved + `/etc/pve/nodes/` while the cluster still knew the node as ``. Also + refused when the check cannot be read. A single node renames as before (#21). - `apply_updates()` reports a failed upgrade as `success: False`, with apt-get's exit status in `error`; it reported success whatever apt-get did (#20). - `get_users()` lists the node's root and regular login accounts (uid 1000 and diff --git a/napalm_proxmox/system_mixin.py b/napalm_proxmox/system_mixin.py index 669f078..2a013bf 100644 --- a/napalm_proxmox/system_mixin.py +++ b/napalm_proxmox/system_mixin.py @@ -26,6 +26,7 @@ import re import shlex from typing import Any +from napalm.base.exceptions import CommandErrorException from napalm_device_types import APT_UPGRADABLE_COMMAND, parse_apt_upgradable from napalm_proxmox.ntp import NTP_STATS_COMMAND, parse_ntp_stats @@ -49,6 +50,11 @@ _PACKAGE_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_.+:~-]*$") _SNMP_WORD = re.compile(r'^[^\s"#]+$') #: The net-snmp agent's configuration file. _SNMPD_CONF = "/etc/snmp/snmpd.conf" +#: Exit status 0 when the node is a cluster member. pvecm itself decides +#: membership by /etc/pve/corosync.conf; /etc/corosync/corosync.conf is the +#: local copy every member keeps, there even when /etc/pve is not mounted. +#: Not ``pvecm status``: it also fails on a member whose corosync is down. +_CLUSTER_MEMBER_CHECK = "[ -e /etc/pve/corosync.conf ] || [ -e /etc/corosync/corosync.conf ]" def _rename_in_hosts(hosts: str, old_short: str, new_short: str, new_domain: str | None) -> str: @@ -617,6 +623,18 @@ class ProxmoxSystemMixin: "snmpd enabled and running", ) + def _refuse_clustered_node(self) -> None: + _, status = self._rc(_CLUSTER_MEMBER_CHECK) + if status == 0: + raise CommandErrorException( + "set_hostname refused: the node is part of a cluster; rename a cluster" + " member by Proxmox's own procedure, not in place" + ) + if status != 1: + raise CommandErrorException( + "set_hostname refused: could not tell whether the node is part of a cluster" + ) + def set_hostname(self, new_hostname: str) -> None: """Set the system hostname on the Proxmox node. @@ -634,16 +652,16 @@ class ProxmoxSystemMixin: 7. Proxmox node TLS certificate — regenerated via ``pvenode cert create --overwrite``. - **Important**: the Proxmox *node name* (shown in the web UI and stored - in ``/etc/pve/nodes//``) only changes after a full reboot. - Until then the web UI still shows the old node name, but the OS-level - hostname and the SSL certificate already reflect the new value. + The Proxmox *node name* (web UI, ``/etc/pve/nodes//``) only + changes after a reboot. Accepts a bare hostname or an FQDN. - Accepts a bare hostname (``pve-see``) or an FQDN - (``pve-see.see.local``). + Raises CommandErrorException, before changing anything, on a cluster + member (a corosync.conf exists): its rename is the cluster's procedure. """ import base64 as _b64 + self._refuse_clustered_node() + if "." in new_hostname: short_hostname, domain = new_hostname.split(".", 1) fqdn = new_hostname diff --git a/tests/test_set_hostname.py b/tests/test_set_hostname.py index cc59e31..b5f86ab 100644 --- a/tests/test_set_hostname.py +++ b/tests/test_set_hostname.py @@ -1,4 +1,5 @@ -"""set_hostname: what it writes to /etc/hosts (napalm-proxmox#21).""" +"""set_hostname: what it writes to /etc/hosts, and that it leaves a clustered node alone +(napalm-proxmox#21).""" from __future__ import annotations @@ -6,6 +7,7 @@ import base64 import re import pytest +from napalm.base.exceptions import CommandErrorException from napalm_proxmox import ProxmoxDriver @@ -18,13 +20,20 @@ HOSTS = """\ class _Node: - """The node's shell: /etc/hostname reads "pve", /etc/hosts reads HOSTS.""" + """The node's shell: /etc/hostname reads "pve", /etc/hosts reads HOSTS. + + *cluster_check* is what the test for a corosync.conf prints: exit status 1 + (no corosync.conf, a single node) unless a test says otherwise. + """ def __init__(self) -> None: self.ran: list[str] = [] + self.cluster_check = "__PVE_RC=1" def __call__(self, command: str) -> str: self.ran.append(command) + if "corosync.conf" in command: + return self.cluster_check if command.startswith("cat /etc/hostname"): return "pve\n" if command.startswith("cat /etc/hosts"): @@ -70,3 +79,29 @@ def test_the_same_name_leaves_hosts_alone(driver, node): driver.set_hostname("pve") assert not any("/etc/hosts" in c and "base64 -d" in c for c in node.ran) + + +def test_a_clustered_node_is_refused_before_anything_changes(driver, node): + node.cluster_check = "__PVE_RC=0" + + with pytest.raises(CommandErrorException, match="part of a cluster"): + driver.set_hostname("new") + + assert [c for c in node.ran if "corosync.conf" not in c] == [] + + +def test_the_cluster_check_looks_at_both_corosync_confs(driver, node): + driver.set_hostname("new") + + check = next(c for c in node.ran if "corosync.conf" in c) + assert "/etc/pve/corosync.conf" in check + assert "/etc/corosync/corosync.conf" in check + + +def test_an_unreadable_cluster_check_refuses_too(driver, node): + node.cluster_check = "" + + with pytest.raises(CommandErrorException, match="could not tell"): + driver.set_hostname("new") + + assert [c for c in node.ran if "corosync.conf" not in c] == [] From 0704d9a102e6f77da313a9abf4b3e2c38a57ac94 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 22:15:01 +0100 Subject: [PATCH 14/14] feat: HA state, backups and votes in get_clusters(); migrate_vm, preflight, maintenance get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool members, the newest backup per guest on each backup storage (PBS included) and /cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a guest of the connected node and follows the task (HA-managed guests through the HA manager), raising with the task log's ERROR lines; get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance() runs ha-manager over SSH. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761). --- CHANGELOG.md | 25 ++ napalm_proxmox/cluster_backups.py | 151 ++++++++ napalm_proxmox/cluster_ha.py | 105 ++++++ napalm_proxmox/cluster_mixin.py | 39 +- napalm_proxmox/driver.py | 2 + napalm_proxmox/migration.py | 115 ++++++ napalm_proxmox/pve_values.py | 27 ++ napalm_proxmox/vm_migrate_mixin.py | 255 +++++++++++++ tests/test_cluster_ha_backups.py | 555 +++++++++++++++++++++++++++++ tests/test_clusters.py | 14 + tests/test_vm_migration.py | 382 ++++++++++++++++++++ 11 files changed, 1653 insertions(+), 17 deletions(-) create mode 100644 napalm_proxmox/cluster_backups.py create mode 100644 napalm_proxmox/cluster_ha.py create mode 100644 napalm_proxmox/migration.py create mode 100644 napalm_proxmox/pve_values.py create mode 100644 napalm_proxmox/vm_migrate_mixin.py create mode 100644 tests/test_cluster_ha_backups.py create mode 100644 tests/test_vm_migration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ff980..4903d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). ### Added +- `get_clusters()` reports quorum `votes` per node (`/cluster/config/nodes`), the + guests HA manages in `ha_resources` (`/cluster/ha/resources`, the CRM state and + node from `/cluster/ha/status/current`, the ids of the PVE 9 HA rules that name + each from `/cluster/ha/rules`, `[]` on PVE 8), the vzdump jobs in `backup_jobs` + (`/cluster/backup`; a pool job lists the pool's guests from `/pools?poolid=`, or + `/pools/{pool}` before PVE 8.1; old `starttime`/`dow` jobs get the schedule PVE + itself derives) and per guest and storage the newest backup in `backups` (each + storage with backup content read once if shared, on each node if local; PBS + included). Each is best-effort: one that cannot be read is None and the rest of + the report stands. +- `get_vm_migration_preflight(name)`: a VM's nodes and blockers from + `GET .../qemu/{vmid}/migrate` in plain words, a local ISO in a CD drive as a + blocking resource, the cloud-init drive left out; a container's candidates are the + online members, its local volumes and unshared bind mounts read from its config. + An offline member is refused as `"node offline"` for both. +- `migrate_vm(name, target_node, online=True, target_storage=None, timeout=3600)`: + `POST .../{qemu|lxc}/{vmid}/migrate` (`online` for a running VM, `with-local-disks` + when it has local disks, restart mode for a running container), followed to the + end; a failure raises with the task log's `ERROR:` lines. A target that is not + another cluster member is refused before the API is called. A guest HA manages is + followed through the HA manager until it is on the target. +- `set_node_maintenance(node, enabled)`: `ha-manager crm-command node-maintenance` + over the driver's SSH session, for a cluster member only and quoted; raises + RuntimeError when SSH is not available. Needs the shapes and contracts of the + next napalm-device-types release for type checking (runtime imports none). - `get_clusters()`: the cluster the node belongs to, always exactly one, read from `/cluster/status`, `/cluster/resources`, each online node's status and network, SDN vnets/zones and the HA manager. Every member is listed, an diff --git a/napalm_proxmox/cluster_backups.py b/napalm_proxmox/cluster_backups.py new file mode 100644 index 0000000..4e1a8ae --- /dev/null +++ b/napalm_proxmox/cluster_backups.py @@ -0,0 +1,151 @@ +"""get_clusters() for Proxmox VE: the backup jobs, and the backups on the storages. + +Read from: + +- ``GET /cluster/backup``: each vzdump job. ``enabled`` absent means enabled; + old jobs carry ``starttime``/``dow`` instead of ``schedule``, which PVE itself + turns into ``" "`` (``PVE/API2/Backup.pm``). ``storage`` and + ``mode`` are None where the job does not set them: a node's + ``/etc/vzdump.conf`` then decides. +- a pool job's guests: ``GET /pools?poolid=`` (PVE 8.1 and later), else + ``GET /pools/``, which older releases have; +- ``GET /nodes//storage//content?content=backup`` for each storage + that holds backups (PBS answers the same): a shared one once, on the own node + if it sees it, else on the next node that does; a local one on every node + that has it. + +Each list is best-effort: when part of it cannot be read it is None, because +half a list would say a guest has no backup, or a job no guest, when it may. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from napalm_proxmox.pve_values import split_list, to_flag, to_int, vmid_list + +if TYPE_CHECKING: + # Type-only: newer than the napalm_device_types floor in pyproject.toml. + from napalm_device_types import BackupDict, BackupJobDict, ClusterStorageDict + +logger = logging.getLogger(__name__) + +_JsonDict = dict[str, Any] + +_ALL_DAYS = "mon,tue,wed,thu,fri,sat,sun" +_GUEST_TYPES = ("qemu", "lxc") + + +def _schedule(job: _JsonDict) -> str | None: + if job.get("schedule"): + return str(job["schedule"]) + start = job.get("starttime") + if not start: + return None + dow = ",".join(split_list(job.get("dow"))) + return str(start) if dow in ("", _ALL_DAYS) else f"{dow} {start}" + + +def _pool_info(api: Any, pool: str) -> _JsonDict: + try: + for entry in api.pools.get(poolid=pool) or []: + if entry.get("poolid") == pool: + return dict(entry) + except Exception as exc: + logger.debug("GET /pools?poolid=%s failed, trying /pools/%s: %s", pool, pool, exc) + return dict(api.pools(pool).get() or {}) + + +def _pool_members(api: Any, pool: str) -> list[int]: + members = _pool_info(api, pool).get("members") or [] + return [ + int(m["vmid"]) + for m in members + if m.get("type") in _GUEST_TYPES and m.get("vmid") is not None + ] + + +def _job(api: Any, job: _JsonDict) -> BackupJobDict: + pool = job.get("pool") or None + return { + "id": str(job.get("id") or ""), + "enabled": to_flag(job.get("enabled", 1)), + "schedule": _schedule(job), + "storage": job.get("storage") or None, + "mode": job.get("mode") or None, + "all": to_flag(job.get("all")), + "vmids": _pool_members(api, str(pool)) if pool else vmid_list(job.get("vmid")), + "exclude": vmid_list(job.get("exclude")), + "pool": pool, + "node": job.get("node") or None, + } + + +def read_backup_jobs(api: Any) -> list[BackupJobDict] | None: + """The cluster's backup jobs; None when they, or a pool's guests, cannot be read.""" + try: + return [_job(api, job) for job in api.cluster.backup.get() or []] + except Exception as exc: + logger.debug("Cannot read the backup jobs: %s", exc) + return None + + +def _content(api: Any, node: str, storage: str) -> list[_JsonDict]: + return list(api.nodes(node).storage(storage).content.get(content="backup") or []) + + +def _first_answer(api: Any, storage: str, nodes: list[str]) -> list[_JsonDict]: + error: Exception | None = None + for node in nodes: + try: + return _content(api, node, storage) + except Exception as exc: + logger.debug("Cannot read backups on %s from %s: %s", storage, node, exc) + error = exc + if error is not None: + raise error + return [] + + +def _storage_rows(api: Any, storage: ClusterStorageDict, local_node: str) -> list[_JsonDict]: + nodes = storage["nodes"] + if storage["shared"]: + return _first_answer(api, storage["name"], sorted(nodes, key=lambda n: n != local_node)) + return [row for node in nodes for row in _content(api, node, storage["name"])] + + +def _merge(rows: list[_JsonDict], storage: str, merged: dict[tuple[int, str], BackupDict]) -> None: + for row in rows: + if row.get("vmid") is None or row.get("ctime") is None: + continue # not a vzdump/PBS backup of a guest + key, ctime = (int(row["vmid"]), storage), int(row["ctime"]) + entry = merged.get(key) + if entry is None: + merged[key] = { + "vmid": key[0], + "storage": storage, + "time": ctime, + "size": to_int(row.get("size")), + "count": 1, + } + continue + entry["count"] += 1 + if ctime > entry["time"]: + entry["time"], entry["size"] = ctime, to_int(row.get("size")) + + +def read_backups( + api: Any, storages: list[ClusterStorageDict], local_node: str +) -> list[BackupDict] | None: + """One entry per guest and storage with its newest backup; None if a storage + that holds backups cannot be read.""" + merged: dict[tuple[int, str], BackupDict] = {} + try: + for storage in storages: + if "backup" in storage["content"]: + _merge(_storage_rows(api, storage, local_node), storage["name"], merged) + except Exception as exc: + logger.debug("Cannot read the backups: %s", exc) + return None + return [merged[key] for key in sorted(merged, key=lambda k: (k[1], k[0]))] diff --git a/napalm_proxmox/cluster_ha.py b/napalm_proxmox/cluster_ha.py new file mode 100644 index 0000000..671dfb4 --- /dev/null +++ b/napalm_proxmox/cluster_ha.py @@ -0,0 +1,105 @@ +"""get_clusters() for Proxmox VE: quorum votes and the guests the HA manager keeps. + +Read from: + +- ``GET /cluster/config/nodes``: corosync's nodelist, ``quorum_votes`` per node; +- ``GET /cluster/ha/resources``: each HA resource's ``sid`` (``vm:100``, + ``ct:101``), requested ``state`` and, on PVE 8, ``group``; +- ``GET /cluster/ha/status/current``: per ``service`` entry the ``node`` and the + CRM's ``crm_state`` (only the verbose ``state`` while the CRM has not taken it + up yet); +- ``GET /cluster/ha/rules`` (PVE 9): each rule's ``rule`` id and its + ``resources``, a comma-separated list of sids. PVE 8 has no such endpoint. + +Each read is best-effort: what cannot be read is None, never a guess. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from napalm_proxmox.pve_values import split_list + +if TYPE_CHECKING: + # Type-only: newer than the napalm_device_types floor in pyproject.toml. + from napalm_device_types import HAResourceDict + +logger = logging.getLogger(__name__) + +_JsonDict = dict[str, Any] + +#: HA resource type prefix -> guest kind. +_KINDS = {"vm": "qemu", "ct": "lxc"} +#: corosync counts a node without ``quorum_votes`` as one vote. +_DEFAULT_VOTES = 1 +#: What HA keeps a resource at when its config names no state. +_DEFAULT_REQUESTED = "started" + + +def read_votes(api: Any) -> dict[str, int] | None: + """Quorum votes per node name; None when the nodelist cannot be read.""" + try: + rows = api.cluster.config.nodes.get() or [] + except Exception as exc: + logger.debug("Cannot read the corosync nodelist: %s", exc) + return None + return { + str(row.get("node") or row.get("name")): int(row.get("quorum_votes", _DEFAULT_VOTES)) + for row in rows + if row.get("node") or row.get("name") + } + + +def _services(api: Any) -> dict[str, _JsonDict] | None: + try: + rows = api.cluster.ha.status.current.get() or [] + except Exception as exc: + logger.debug("Cannot read the HA status: %s", exc) + return None + return {str(r["sid"]): r for r in rows if r.get("type") == "service" and r.get("sid")} + + +def _rules(api: Any) -> dict[str, list[str]]: + """Rule ids per sid. PVE 8 answers with an error: no rules there.""" + try: + rows = api.cluster.ha.rules.get() or [] + except Exception as exc: + logger.debug("No HA rules (PVE 8 has none): %s", exc) + return {} + by_sid: dict[str, list[str]] = {} + for rule in rows: + for sid in split_list(rule.get("resources")): + by_sid.setdefault(sid, []).append(str(rule.get("rule"))) + return by_sid + + +def _resource( + row: _JsonDict, services: dict[str, _JsonDict] | None, rules: dict[str, list[str]] +) -> HAResourceDict | None: + sid = str(row.get("sid") or "") + prefix, _, vmid = sid.partition(":") + if prefix not in _KINDS or not vmid.isdigit(): + return None + service = (services or {}).get(sid, {}) + return { + "vmid": int(vmid), + "kind": _KINDS[prefix], + "requested": str(row.get("state") or _DEFAULT_REQUESTED), + "status": service.get("crm_state") or service.get("state") or None, + "node": service.get("node") or None, + "group": row.get("group") or None, + "rules": rules.get(sid, []), + } + + +def read_ha_resources(api: Any) -> list[HAResourceDict] | None: + """Every guest HA manages, by vmid; None when the HA resources cannot be read.""" + try: + rows = api.cluster.ha.resources.get() or [] + except Exception as exc: + logger.debug("Cannot read the HA resources: %s", exc) + return None + services, rules = _services(api), _rules(api) + resources = [_resource(row, services, rules) for row in rows] + return sorted((r for r in resources if r is not None), key=lambda r: r["vmid"]) diff --git a/napalm_proxmox/cluster_mixin.py b/napalm_proxmox/cluster_mixin.py index 73afccd..ca9fc2a 100644 --- a/napalm_proxmox/cluster_mixin.py +++ b/napalm_proxmox/cluster_mixin.py @@ -13,7 +13,10 @@ cluster of one, without a name). Read from: - ``GET /nodes/{node}/network`` of each online node: its bridges; - ``GET /cluster/sdn/vnets`` and ``/cluster/sdn/zones``: the vnets, on the nodes of their zone; -- ``GET /cluster/ha/status/manager_status``: which node is in maintenance. +- ``GET /cluster/ha/status/manager_status``: which node is in maintenance; +- quorum votes and HA resources: see ``cluster_ha``; backup jobs and backups: + see ``cluster_backups``. Each of those is best-effort and None when it cannot + be read, without failing the rest of the report. """ from __future__ import annotations @@ -21,7 +24,12 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any, NamedTuple +from napalm_proxmox.cluster_backups import read_backup_jobs, read_backups +from napalm_proxmox.cluster_ha import read_ha_resources, read_votes from napalm_proxmox.network_targets import bridge_targets, vnet_target +from napalm_proxmox.pve_values import split_list as _list +from napalm_proxmox.pve_values import to_flag as _flag +from napalm_proxmox.pve_values import to_int as _int from napalm_proxmox.utils import cpu_flags if TYPE_CHECKING: @@ -51,22 +59,10 @@ class _Cpu(NamedTuple): flags: list[str] | None = None -def _int(value: Any) -> int | None: - return None if value is None else int(value) - - def _float(value: Any) -> float | None: return None if value is None else float(value) -def _flag(value: Any) -> bool: - return bool(int(value or 0)) - - -def _list(value: Any) -> list[str]: - return [part.strip() for part in str(value or "").split(",") if part.strip()] - - def _size(rows: list[_JsonDict], key: str, shared: bool) -> int | None: """One storage's size over the nodes in *rows*: summed unless it is shared.""" values = [row.get(key) for row in rows] @@ -151,14 +147,18 @@ class ProxmoxClusterMixin: resources = self._api.cluster.resources.get() or [] head = next((s for s in status if s.get("type") == "cluster"), None) members = self._members(status if head else []) + storages = _storages(resources) return [ { "id": None, "name": (head.get("name") or None) if head else None, "quorate": _flag(head.get("quorate")) if head else None, "nodes": self._cluster_nodes(members, resources), - "storages": _storages(resources), + "storages": storages, "networks": self._networks(members), + "ha_resources": read_ha_resources(self._api), + "backup_jobs": read_backup_jobs(self._api), + "backups": read_backups(self._api, storages, self._node_name), } ] @@ -166,8 +166,8 @@ class ProxmoxClusterMixin: self, members: list[_Member], resources: list[_JsonDict] ) -> list[ClusterNodeDict]: rows = {r.get("node"): r for r in resources if r.get("type") == "node"} - maintenance = self._maintenance() - return [self._cluster_node(m, rows.get(m.name, {}), maintenance) for m in members] + maintenance, votes = self._maintenance(), read_votes(self._api) + return [self._cluster_node(m, rows.get(m.name, {}), maintenance, votes) for m in members] def _networks(self, members: list[_Member]) -> list[ClusterNetworkDict]: online = [m for m in members if m.online] @@ -225,7 +225,11 @@ class ProxmoxClusterMixin: ) def _cluster_node( - self, member: _Member, row: _JsonDict, maintenance: dict[str, bool] | None + self, + member: _Member, + row: _JsonDict, + maintenance: dict[str, bool] | None, + votes: dict[str, int] | None, ) -> ClusterNodeDict: if not member.online: row, cpu = {}, _Cpu() @@ -243,6 +247,7 @@ class ProxmoxClusterMixin: "cpu_usage": _float(row.get("cpu")), "memory_total": _int(row.get("maxmem")), "memory_used": _int(row.get("mem")), + "votes": None if votes is None else votes.get(member.name), } def _vnets(self, members: list[str]) -> list[ClusterNetworkDict]: diff --git a/napalm_proxmox/driver.py b/napalm_proxmox/driver.py index 8ca45dc..fc2eb27 100644 --- a/napalm_proxmox/driver.py +++ b/napalm_proxmox/driver.py @@ -63,6 +63,7 @@ from napalm_proxmox.routing_mixin import ProxmoxRoutingMixin from napalm_proxmox.sdn_mixin import ProxmoxSDNMixin from napalm_proxmox.system_mixin import ProxmoxSystemMixin from napalm_proxmox.vm_contract_mixin import ProxmoxVMContractMixin +from napalm_proxmox.vm_migrate_mixin import ProxmoxVMMigrateMixin from napalm_proxmox.vm_mixin import ProxmoxVMMixin from napalm_proxmox.vm_provision_mixin import ProxmoxVMProvisionMixin from napalm_proxmox.vm_snapshot_mixin import ProxmoxVMSnapshotMixin @@ -89,6 +90,7 @@ class ProxmoxDriver( ProxmoxVMSnapshotMixin, ProxmoxVMProvisionMixin, ProxmoxClusterMixin, + ProxmoxVMMigrateMixin, ProxmoxRoutingMixin, ProxmoxSystemMixin, KernelFactsMixin, diff --git a/napalm_proxmox/migration.py b/napalm_proxmox/migration.py new file mode 100644 index 0000000..d92925a --- /dev/null +++ b/napalm_proxmox/migration.py @@ -0,0 +1,115 @@ +"""Reading Proxmox VE's answers about a guest's migration: refusal reasons, local +volumes and resources, task log errors. + +- ``GET /nodes/{node}/qemu/{vmid}/migrate`` (qemu-server's + ``migrate_vm_precondition``): ``not_allowed_nodes`` maps a node to its + blockers -- ``unavailable_storages``, ``unavailable-resources`` (mappings the + node lacks), ``blocking-ha-resources`` (PVE 9 HA rules); ``local_disks`` lists + volumes on non-shared storage, CD drives included; ``local_resources`` + (``hostpci0``, ``usb0``, ``clipboard=vnc``) block the migration. +- A container has no such endpoint; its config says what it keeps locally + (``PVE/LXC/Migrate.pm``): a bind or device mount point that is not marked + ``shared`` stops the migration, a volume on non-shared storage is copied. +- A task log is rows of ``{"n": , "t": }``; a failure writes + ``ERROR: ...`` lines. +""" + +from __future__ import annotations + +import re +from typing import Any + +from napalm_proxmox.pve_values import to_flag + +_JsonDict = dict[str, Any] + +_ERROR = re.compile(r"\bERROR: (.*\S)") +#: The cloud-init drive is a CD drive the migration regenerates on the target. +_CLOUDINIT = re.compile(r"vm-\d+-cloudinit") +_CT_VOLUME_KEY = re.compile(r"^(rootfs|mp\d+|unused\d+)$") + + +def task_errors(log: list[_JsonDict]) -> list[str]: + """The ``ERROR:`` lines of a task log, without their timestamps, each once.""" + errors: list[str] = [] + for row in log: + match = _ERROR.search(str(row.get("t") or "")) + if match and match.group(1) not in errors: + errors.append(match.group(1)) + return errors + + +def upid_field(upid: str, index: int) -> str: + """A field of ``UPID:node:pid:pstart:starttime:type:id:user:``; "" if absent.""" + parts = upid.split(":") + return parts[index] if len(parts) > index else "" + + +def _names(kind: str, items: list[Any]) -> str: + names = ", ".join(str(item) for item in items) + return f"{kind} {names} is" if len(items) == 1 else f"{kind}s {names} are" + + +def _ha_block(item: Any) -> str: + if not isinstance(item, dict): + return f"HA resource {item} blocks it" + cause = f" ({item['cause']} rule)" if item.get("cause") else "" + return f"HA resource {item.get('sid')} blocks it{cause}" + + +def _reason(key: str, value: Any) -> str: + items = value if isinstance(value, list) else [value] + if key == "unavailable_storages": + return f"{_names('storage', items)} not available there" + if key == "unavailable-resources": + return f"{_names('mapped resource', items)} not available there" + if key == "blocking-ha-resources": + return "; ".join(_ha_block(item) for item in items) + return f"{key}: {', '.join(str(item) for item in items)}" + + +def refusal(blockers: Any) -> str: + """One node's ``not_allowed_nodes`` entry in plain words.""" + if not isinstance(blockers, dict) or not blockers: + return "refused by Proxmox VE" + return "; ".join(_reason(key, blockers[key]) for key in sorted(blockers)) + + +def qemu_local(answer: _JsonDict) -> tuple[list[str], list[str]]: + """``(local_disks, local_resources)`` of a VM's migrate precondition answer. + + A local ISO in a CD drive stops the migration, so it is a resource + (``cdrom=``, after PVE's own ``clipboard=vnc``), not a disk. + """ + disks: list[str] = [] + resources = [str(r) for r in answer.get("local_resources") or []] + for disk in answer.get("local_disks") or []: + volid = str(disk.get("volid") or "") + if not volid or _CLOUDINIT.search(volid): + continue + if to_flag(disk.get("cdrom")): + resources.append(f"cdrom={volid}") + else: + disks.append(volid) + return sorted(disks), resources + + +def lxc_local(config: _JsonDict, shared: dict[str, bool]) -> tuple[list[str], list[str]]: + """``(local_disks, local_resources)`` of a container's config. + + *shared* says per storage whether it is shared; a storage it does not know is + taken as local, which blocks nothing. + """ + disks: list[str] = [] + resources: list[str] = [] + for key in sorted(config): + if not _CT_VOLUME_KEY.match(key): + continue + volume, *options = str(config[key]).split(",") + volume = volume.removeprefix("volume=") + if volume.startswith("/"): + if "shared=1" not in options: + resources.append(key) + elif not shared.get(volume.split(":", 1)[0], False): + disks.append(volume) + return sorted(disks), resources diff --git a/napalm_proxmox/pve_values.py b/napalm_proxmox/pve_values.py new file mode 100644 index 0000000..d64163d --- /dev/null +++ b/napalm_proxmox/pve_values.py @@ -0,0 +1,27 @@ +"""How the Proxmox VE API writes plain values: numbers as strings, flags as 0/1, lists +as comma-separated strings.""" + +from __future__ import annotations + +import re +from typing import Any + +_LIST_SEPARATORS = re.compile(r"[\s,;\0]+") + + +def to_int(value: Any) -> int | None: + return None if value is None else int(value) + + +def to_flag(value: Any) -> bool: + return bool(int(value or 0)) + + +def split_list(value: Any) -> list[str]: + """``"a,b"`` -> ``["a", "b"]``; PVE also separates with ``;``, blanks or NUL.""" + return [part for part in _LIST_SEPARATORS.split(str(value or "")) if part] + + +def vmid_list(value: Any) -> list[int]: + """``"100,101"`` -> ``[100, 101]``; what is not a number is left out.""" + return [int(part) for part in split_list(value) if part.isdigit()] diff --git a/napalm_proxmox/vm_migrate_mixin.py b/napalm_proxmox/vm_migrate_mixin.py new file mode 100644 index 0000000..752ac94 --- /dev/null +++ b/napalm_proxmox/vm_migrate_mixin.py @@ -0,0 +1,255 @@ +"""HypervisorDriver migration methods for Proxmox VE: preflight, migrate, HA maintenance. + +A guest is addressed like every other VM method of this driver: by vmid or name, +on the node the driver is connected to (``_resolve_vm``). + +- ``get_vm_migration_preflight``: a VM asks ``GET .../qemu/{vmid}/migrate``; a + container has no such endpoint, so the online members are its candidates. + A member the cluster calls offline is refused (``"node offline"``) for both, + also where the VM endpoint would list it as allowed. +- ``migrate_vm``: ``POST .../{qemu|lxc}/{vmid}/migrate`` and the task is followed + to its end; a failure raises with the task log's ``ERROR:`` lines. A guest HA + manages migrates through a ``hamigrate`` task that only hands the request to + the HA manager; the driver then follows ``/cluster/ha/status/current`` until + the guest is on the target, and on a failure reads the migration task's log. +- ``set_node_maintenance``: PVE has no REST endpoint for it; it runs + ``ha-manager crm-command node-maintenance`` over the driver's SSH session. + The HA manager moves the node's HA guests away after the command returned. +""" + +from __future__ import annotations + +import logging +import shlex +from time import monotonic, sleep +from typing import TYPE_CHECKING, Any + +from napalm_proxmox.migration import lxc_local, qemu_local, refusal, task_errors, upid_field +from napalm_proxmox.pve_values import to_flag + +if TYPE_CHECKING: + # Type-only: newer than the napalm_device_types floor in pyproject.toml. + from napalm_device_types import MigrationPreflightDict + +logger = logging.getLogger(__name__) + +_JsonDict = dict[str, Any] + +#: Enough to reach the ERROR lines at the end of a long storage migration's log. +_TASK_LOG_LINES = 50000 +_HA_POLL_SECONDS = 5 +#: CRM states while the HA manager is moving a guest. +_HA_MOVING = frozenset({"migrate", "relocate"}) + + +class ProxmoxVMMigrateMixin: + """Relies on the driver, ``_resolve_vm``/``_guest_api``, ``_members`` and + ``_wait_for_task``/``_run_node_command``/``_node_ssh``.""" + + _api: Any + _node_name: str + hostname: str + timeout: int + _node_api: Any + _resolve_vm: Any + _guest_api: Any + _members: Any + _wait_for_task: Any + _node_ssh: Any + _run_node_command: Any + + # -- cluster membership ------------------------------------------------------- + + def _cluster_members(self) -> list[Any]: + status = self._api.cluster.status.get() or [] + clustered = any(s.get("type") == "cluster" for s in status) + return list(self._members(status if clustered else [])) + + def _other_member(self, node: str) -> None: + names = [m.name for m in self._cluster_members()] + if node == self._node_name or node not in names: + raise ValueError( + f"{node!r} is not another member of the cluster of {self._node_name!r} " + f"(members: {', '.join(names)})" + ) + + def _storage_sharing(self) -> dict[str, bool]: + rows = self._api.cluster.resources.get(type="storage") or [] + sharing: dict[str, bool] = {} + for row in rows: + if row.get("type") == "storage" and row.get("storage"): + name = str(row["storage"]) + sharing[name] = sharing.get(name, False) or to_flag(row.get("shared")) + return sharing + + # -- preflight ---------------------------------------------------------------- + + def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict: + """Where the guest could move to within its cluster, and what blocks it.""" + vmid, vm_type = self._resolve_vm(name) + others = [m for m in self._cluster_members() if m.name != self._node_name] + offline = [m.name for m in others if not m.online] + if vm_type == "vm": + return self._qemu_preflight(vmid, offline) + return self._lxc_preflight(vmid, [m.name for m in others if m.online], offline) + + def _qemu_preflight(self, vmid: int, offline: list[str]) -> MigrationPreflightDict: + answer = self._guest_api(vmid, "vm").migrate.get() or {} + refused = { + str(node): refusal(blockers) + for node, blockers in (answer.get("not_allowed_nodes") or {}).items() + } + for node in offline: + refused[node] = "; ".join(filter(None, ["node offline", refused.get(node)])) + disks, resources = qemu_local(answer) + return { + "running": to_flag(answer.get("running")), + "allowed_nodes": [ + str(n) for n in answer.get("allowed_nodes") or [] if n not in offline + ], + "refused_nodes": refused, + "local_disks": disks, + "local_resources": resources, + } + + def _lxc_running(self, vmid: int) -> bool: + status = self._guest_api(vmid, "container").status.current.get() or {} + return bool(status.get("status") == "running") + + def _lxc_preflight( + self, vmid: int, online: list[str], offline: list[str] + ) -> MigrationPreflightDict: + config = self._guest_api(vmid, "container").config.get() or {} + disks, resources = lxc_local(config, self._storage_sharing()) + return { + "running": self._lxc_running(vmid), + "allowed_nodes": online, + "refused_nodes": {n: "node offline" for n in offline}, + "local_disks": disks, + "local_resources": resources, + } + + # -- migration ---------------------------------------------------------------- + + def migrate_vm( + self, + name: str, + target_node: str, + online: bool = True, + target_storage: str | None = None, + timeout: int = 3600, + ) -> None: + """Move the guest to *target_node*; block until Proxmox VE finished.""" + vmid, vm_type = self._resolve_vm(name) + self._other_member(target_node) + deadline = monotonic() + timeout + params = self._qemu_params(vmid, online) if vm_type == "vm" else self._lxc_params(vmid) + if target_storage: + params["targetstorage" if vm_type == "vm" else "target-storage"] = target_storage + try: + upid = str(self._guest_api(vmid, vm_type).migrate.post(target=target_node, **params)) + except Exception as exc: + raise RuntimeError( + f"Migration of {vm_type} {vmid} to {target_node!r} refused: {exc}" + ) from exc + self._follow_task(upid, vmid, timeout) + if upid_field(upid, 5) == "hamigrate": + self._follow_ha(vmid, vm_type, target_node, deadline, upid) + + def _qemu_params(self, vmid: int, online: bool) -> dict[str, Any]: + answer = self._guest_api(vmid, "vm").migrate.get() or {} + params: dict[str, Any] = {} + if online and to_flag(answer.get("running")): + params["online"] = 1 + if qemu_local(answer)[0]: + params["with-local-disks"] = 1 + return params + + def _lxc_params(self, vmid: int) -> dict[str, Any]: + # A container cannot move while running; restart mode stops and restarts it. + return {"restart": 1} if self._lxc_running(vmid) else {} + + def _task_errors(self, upid: str) -> list[str]: + try: + log = self._node_api().tasks(upid).log.get(limit=_TASK_LOG_LINES) or [] + except Exception as exc: + logger.debug("Cannot read the log of task %s: %s", upid, exc) + return [] + return task_errors(log) + + def _follow_task(self, upid: str, vmid: int, timeout: int) -> None: + try: + self._wait_for_task(upid, timeout=timeout) + except RuntimeError as exc: + errors = self._task_errors(upid) + if not errors: + raise + raise RuntimeError(f"Migration of guest {vmid} failed: {'; '.join(errors)}") from exc + + # -- a guest HA manages ------------------------------------------------------- + + def _ha_state(self, sid: str) -> tuple[str | None, str | None]: + rows = self._api.cluster.ha.status.current.get() or [] + service: _JsonDict = next((r for r in rows if r.get("sid") == sid), {}) + return service.get("node"), service.get("crm_state") or service.get("state") + + def _follow_ha( + self, vmid: int, vm_type: str, target: str, deadline: float, ha_upid: str + ) -> None: + sid = f"{'vm' if vm_type == 'vm' else 'ct'}:{vmid}" + moving_seen = False + while True: + node, state = self._ha_state(sid) + if node == target and state not in _HA_MOVING: + return + if state == "error": + self._ha_failed(vmid, vm_type, ha_upid, f"HA manager put {sid} into error state") + if state in _HA_MOVING: + moving_seen = True + elif moving_seen: + self._ha_failed( + vmid, vm_type, ha_upid, f"HA manager left {sid} on {node} ({state})" + ) + if monotonic() > deadline: + raise RuntimeError( + f"Migration of {sid} to {target!r} timed out on {node} ({state})" + ) + sleep(_HA_POLL_SECONDS) + + def _ha_failed(self, vmid: int, vm_type: str, ha_upid: str, what: str) -> None: + """Raise *what* with the ERROR lines of the migration the HA manager ran.""" + errors: list[str] = [] + kind = "qmigrate" if vm_type == "vm" else "vzmigrate" + try: + tasks = self._node_api().tasks.get(vmid=vmid, typefilter=kind, limit=1) or [] + since = int(upid_field(ha_upid, 4) or "0", 16) + if tasks and int(tasks[0].get("starttime") or 0) >= since: + errors = self._task_errors(str(tasks[0].get("upid"))) + except Exception as exc: + logger.debug("Cannot find the migration task of guest %s: %s", vmid, exc) + raise RuntimeError(f"{what}: {'; '.join(errors)}" if errors else what) + + # -- HA maintenance ----------------------------------------------------------- + + def set_node_maintenance(self, node: str, enabled: bool) -> None: + """Switch HA maintenance mode of a cluster node, over SSH (no REST endpoint).""" + names = [m.name for m in self._cluster_members()] + if node not in names: + raise ValueError( + f"{node!r} is not a member of the cluster (members: {', '.join(names)})" + ) + action = "enable" if enabled else "disable" + try: + self._node_ssh() + except Exception as exc: + raise RuntimeError( + f"Cannot {action} HA maintenance of node {node!r}: SSH to {self.hostname} " + f"is not available ({exc})" + ) from exc + command = f"ha-manager crm-command node-maintenance {action} {shlex.quote(node)}" + try: + self._run_node_command(command, timeout=self.timeout) + except Exception as exc: + raise RuntimeError( + f"ha-manager refused to {action} maintenance of {node!r}: {exc}" + ) from exc diff --git a/tests/test_cluster_ha_backups.py b/tests/test_cluster_ha_backups.py new file mode 100644 index 0000000..21239bc --- /dev/null +++ b/tests/test_cluster_ha_backups.py @@ -0,0 +1,555 @@ +"""get_clusters(): quorum votes, the HA manager's guests, backup jobs and backups. + +Each of these is read on top of the cluster report and is best-effort: a read +that fails leaves its key None and the rest of the report stands. + +Payloads: ``/cluster/config/nodes`` is corosync's nodelist; ``/cluster/ha/*`` +follow pve-ha-manager's API (``src/PVE/API2/HA/{Resources,Status,Rules}.pm``, +master = PVE 9 and stable-4 = PVE 8): a service entry of ``status/current`` +carries ``crm_state`` and ``request_state``, a rule carries ``rule``, ``type`` +and ``resources`` as a sorted, comma-separated list of sids, and PVE 8 has no +``/cluster/ha/rules``. ``/cluster/backup`` follows pve-manager's +``PVE/API2/Backup.pm``; storage content rows follow +``/nodes/{node}/storage/{storage}/content``. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +from tests.test_clusters import ( + CLUSTER_RESOURCES, + GIB, + _api, + _by_name, + _driver, + _storage_row, +) + + +def _report(api: MagicMock) -> dict[str, Any]: + [cluster] = _driver(api).get_clusters() + return dict(cluster) + + +class TestVotes: + def test_each_node_has_its_quorum_votes(self): + nodes = _by_name(_report(_api())["nodes"]) + assert [nodes[n]["votes"] for n in ("pve1", "pve2", "pve3")] == [1, 1, 2] + + def test_unreadable_nodelist_leaves_votes_unknown(self): + api = _api() + api.cluster.config.nodes.get.side_effect = RuntimeError("403 Forbidden") + nodes = _report(api)["nodes"] + assert [n["votes"] for n in nodes] == [None, None, None] + # The rest of the node still stands. + assert nodes[0]["cpu_threads"] == 32 + + def test_standalone_node_has_no_votes(self): + # A node without corosync.conf answers with an empty nodelist. + api = _api() + api.cluster.config.nodes.get.return_value = [] + assert [n["votes"] for n in _report(api)["nodes"]] == [None, None, None] + + def test_node_without_quorum_votes_counts_the_default(self): + """corosync counts a node without quorum_votes as one vote.""" + api = _api() + api.cluster.config.nodes.get.return_value = [{"node": "pve1", "name": "pve1"}] + nodes = _by_name(_report(api)["nodes"]) + assert nodes["pve1"]["votes"] == 1 + assert nodes["pve2"]["votes"] is None + + +# --------------------------------------------------------------------------- HA + +HA_RESOURCES_PVE8 = [ + { + "sid": "vm:100", + "type": "vm", + "state": "started", + "group": "prefer-pve1", + "max_restart": 1, + "max_relocate": 1, + "digest": "3f1c", + }, + # No state in the config: HA's default, "started". + {"sid": "ct:101", "type": "ct", "digest": "3f1c"}, + {"sid": "vm:102", "type": "vm", "state": "disabled", "digest": "3f1c"}, +] + +HA_STATUS = [ + {"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"}, + { + "id": "master", + "type": "master", + "node": "pve1", + "status": "pve1 (active, Sun Nov 1 10:00:00 2026)", + "timestamp": 1793527200, + }, + { + "id": "lrm:pve1", + "type": "lrm", + "node": "pve1", + "status": "pve1 (active, Sun Nov 1 10:00:00 2026)", + "timestamp": 1793527200, + }, + { + "id": "service:vm:100", + "type": "service", + "sid": "vm:100", + "node": "pve2", + "crm_state": "migrate", + "state": "migrate", + "status": "vm:100 (pve2, migrate)", + "request_state": "started", + "group": "prefer-pve1", + "max_restart": 1, + "max_relocate": 1, + }, + { + "id": "service:ct:101", + "type": "service", + "sid": "ct:101", + "node": "pve1", + "crm_state": "started", + "state": "started", + "status": "ct:101 (pve1, started)", + "request_state": "started", + }, + # Not yet processed by the CRM: no crm_state, a verbose state only. + { + "id": "service:vm:102", + "type": "service", + "sid": "vm:102", + "state": "queued", + "status": "vm:102 (---, queued)", + "request_state": "disabled", + }, +] + +# PVE 9: groups were migrated into node-affinity rules, so resources lose "group". +HA_RESOURCES_PVE9 = [{k: v for k, v in r.items() if k != "group"} for r in HA_RESOURCES_PVE8] + +HA_RULES_PVE9 = [ + { + "rule": "ha-group-prefer-pve1", + "type": "node-affinity", + "resources": "vm:100", + "nodes": "pve1:2,pve2:1", + "strict": 0, + "digest": "9ac2", + "order": 1, + }, + { + "rule": "keep-apart", + "type": "resource-affinity", + "affinity": "negative", + "resources": "ct:101,vm:100", + "digest": "9ac2", + "order": 2, + }, +] + + +def _ha_api( + resources: Any = HA_RESOURCES_PVE8, + status: Any = HA_STATUS, + rules: Any = None, +) -> MagicMock: + api = _api() + ha = api.cluster.ha + for endpoint, value in ((ha.resources, resources), (ha.status.current, status)): + if isinstance(value, Exception): + endpoint.get.side_effect = value + else: + endpoint.get.return_value = value + if rules is None: + # PVE 8 has no rules endpoint. + ha.rules.get.side_effect = RuntimeError( + "501 Not Implemented: Method 'GET /cluster/ha/rules' not implemented" + ) + else: + ha.rules.get.return_value = rules + return api + + +def _ha(api: MagicMock) -> dict[int, dict[str, Any]] | None: + resources = _report(api)["ha_resources"] + return None if resources is None else {r["vmid"]: r for r in resources} + + +class TestHAResources: + def test_pve8_resource_with_group(self): + ha = _ha(_ha_api()) + assert ha is not None + assert ha[100] == { + "vmid": 100, + "kind": "qemu", + "requested": "started", + "status": "migrate", + "node": "pve2", + "group": "prefer-pve1", + "rules": [], + } + + def test_container_and_default_requested_state(self): + ha = _ha(_ha_api()) + assert ha is not None + assert ha[101]["kind"] == "lxc" + assert ha[101]["requested"] == "started" + assert ha[101]["status"] == "started" + assert ha[101]["node"] == "pve1" + assert ha[101]["group"] is None + + def test_not_yet_processed_reports_the_verbose_state(self): + ha = _ha(_ha_api()) + assert ha is not None + assert ha[102]["requested"] == "disabled" + assert ha[102]["status"] == "queued" + assert ha[102]["node"] is None + + def test_pve9_rules_name_their_resources(self): + ha = _ha(_ha_api(resources=HA_RESOURCES_PVE9, rules=HA_RULES_PVE9)) + assert ha is not None + assert ha[100]["rules"] == ["ha-group-prefer-pve1", "keep-apart"] + assert ha[101]["rules"] == ["keep-apart"] + assert ha[102]["rules"] == [] + assert ha[100]["group"] is None + + def test_no_ha_configured_is_an_empty_list(self): + assert _report(_ha_api(resources=[], status=[]))["ha_resources"] == [] + + def test_unreadable_resources_leave_the_list_unknown(self): + api = _ha_api(resources=RuntimeError("403 Forbidden: Permission check failed")) + report = _report(api) + assert report["ha_resources"] is None + assert report["name"] == "lab" + + def test_unreadable_status_leaves_status_and_node_unknown(self): + ha = _ha(_ha_api(status=RuntimeError("500 cfs lock"))) + assert ha is not None + assert (ha[100]["status"], ha[100]["node"]) == (None, None) + assert ha[100]["requested"] == "started" + + +# ---------------------------------------------------------------------- backups + +BACKUP_JOBS = [ + { + "id": "backup-5a1b2c3d-e4f5", + "type": "vzdump", + "enabled": 1, + "schedule": "sat 02:00", + "storage": "pbs", + "mode": "snapshot", + "all": 1, + "exclude": "105,106", + "compress": "zstd", + "mailnotification": "always", + "next-run": 1793584800, + "notes-template": "{{guestname}}", + }, + { + "id": "backup-7e8f9a0b-1c2d", + "type": "vzdump", + "enabled": 0, + "schedule": "mon..fri 21:00", + "storage": "local", + "mode": "stop", + "vmid": "100,101", + "node": "pve1", + }, + # No "enabled": a job is enabled unless it says otherwise. + { + "id": "backup-0b1c2d3e-4f5a", + "type": "vzdump", + "schedule": "daily", + "storage": "pbs", + "mode": "suspend", + "pool": "prod", + }, +] + +# PVE >= 8.1: GET /pools?poolid=prod +POOL_INDEX = [ + { + "poolid": "prod", + "comment": "", + "members": [ + {"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "name": "web"}, + {"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "pve1", "name": "dns"}, + {"id": "storage/pve1/local", "type": "storage", "storage": "local", "node": "pve1"}, + ], + } +] + + +def _backup_api(jobs: Any = BACKUP_JOBS) -> MagicMock: + api = _api() + if isinstance(jobs, Exception): + api.cluster.backup.get.side_effect = jobs + else: + api.cluster.backup.get.return_value = jobs + api.pools.get.return_value = POOL_INDEX + return api + + +def _jobs(api: MagicMock) -> dict[str, dict[str, Any]]: + jobs = _report(api)["backup_jobs"] + assert jobs is not None + return {j["id"]: j for j in jobs} + + +class TestBackupJobs: + def test_all_guests_job(self): + assert _jobs(_backup_api())["backup-5a1b2c3d-e4f5"] == { + "id": "backup-5a1b2c3d-e4f5", + "enabled": True, + "schedule": "sat 02:00", + "storage": "pbs", + "mode": "snapshot", + "all": True, + "vmids": [], + "exclude": [105, 106], + "pool": None, + "node": None, + } + + def test_disabled_job_for_named_guests_on_one_node(self): + job = _jobs(_backup_api())["backup-7e8f9a0b-1c2d"] + assert job["enabled"] is False + assert job["all"] is False + assert job["vmids"] == [100, 101] + assert job["exclude"] == [] + assert job["node"] == "pve1" + + def test_pool_job_lists_the_pools_guests(self): + api = _backup_api() + job = _jobs(api)["backup-0b1c2d3e-4f5a"] + assert job["enabled"] is True + assert job["pool"] == "prod" + assert job["vmids"] == [100, 101] + api.pools.get.assert_called_with(poolid="prod") + + def test_pool_read_falls_back_to_the_old_endpoint(self): + """Before PVE 8.1, /pools takes no poolid; /pools/{poolid} answers instead.""" + api = _backup_api() + api.pools.get.side_effect = RuntimeError("400 Bad Request: Parameter verification failed") + pool = MagicMock() + pool.get.return_value = {"comment": "", "members": POOL_INDEX[0]["members"]} + api.pools.side_effect = lambda name: pool if name == "prod" else MagicMock() + assert _jobs(api)["backup-0b1c2d3e-4f5a"]["vmids"] == [100, 101] + + def test_old_style_start_time_and_days(self): + """Jobs from vzdump.cron carry starttime/dow; PVE turns them into a schedule.""" + jobs = [ + {"id": "a", "starttime": "02:30", "dow": "sat,sun", "storage": "local", "vmid": "102"}, + { + "id": "b", + "starttime": "03:00", + "dow": "mon,tue,wed,thu,fri,sat,sun", + "storage": "local", + "all": 1, + }, + {"id": "c", "starttime": "04:00", "storage": "local", "all": 1}, + ] + found = _jobs(_backup_api(jobs)) + assert [found[i]["schedule"] for i in "abc"] == ["sat,sun 02:30", "03:00", "04:00"] + + def test_job_with_a_dump_directory_has_no_storage(self): + jobs = [{"id": "d", "schedule": "daily", "dumpdir": "/mnt/backup", "all": 1}] + job = _jobs(_backup_api(jobs))["d"] + assert job["storage"] is None + assert job["mode"] is None + + def test_no_jobs_is_an_empty_list(self): + assert _report(_backup_api([]))["backup_jobs"] == [] + + def test_unreadable_jobs_leave_the_list_unknown(self): + report = _report(_backup_api(RuntimeError("403 Forbidden"))) + assert report["backup_jobs"] is None + assert report["nodes"] + + def test_unreadable_pool_leaves_the_list_unknown(self): + """A pool job whose guests cannot be read would look like a job for none.""" + api = _backup_api() + api.pools.get.side_effect = RuntimeError("403 Forbidden") + api.pools.return_value.get.side_effect = RuntimeError("403 Forbidden") + api.pools.side_effect = None + assert _report(api)["backup_jobs"] is None + + +BACKUP_RESOURCES = [ + *CLUSTER_RESOURCES, + _storage_row("pve1", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB), + _storage_row("pve2", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB), + _storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB), + _storage_row("pve2", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 30 * GIB), + _storage_row("pve3", "local", "dir", 0, "iso,vztmpl,backup", 0, 0, status="unknown"), +] + +PBS_CONTENT = [ + { + "volid": "pbs:backup/vm/100/2026-10-31T01:00:00Z", + "content": "backup", + "format": "pbs-vm", + "ctime": 1793408400, + "size": 32 * GIB, + "vmid": 100, + "subtype": "qemu", + "notes": "web", + "protected": 0, + "verification": {"state": "ok", "upid": "UPID:pbs:0000:0000:0000:verify:x:root@pam:"}, + }, + { + "volid": "pbs:backup/vm/100/2026-10-24T01:00:00Z", + "content": "backup", + "format": "pbs-vm", + "ctime": 1792803600, + "size": 31 * GIB, + "vmid": 100, + "subtype": "qemu", + }, + { + "volid": "pbs:backup/ct/101/2026-10-31T01:05:00Z", + "content": "backup", + "format": "pbs-ct", + "ctime": 1793408700, + "size": 2 * GIB, + "vmid": 101, + "subtype": "lxc", + }, +] + +LOCAL_CONTENT = { + "pve1": [ + { + "volid": "local:backup/vzdump-qemu-100-2026_10_30-21_00_02.vma.zst", + "content": "backup", + "format": "vma.zst", + "ctime": 1793394002, + "size": 1234567, + "vmid": 100, + "subtype": "qemu", + }, + # A file that does not follow vzdump's naming has no vmid. + {"volid": "local:backup/manual.tar", "content": "backup", "format": "tar", "size": 10}, + ], + "pve2": [ + { + "volid": "local:backup/vzdump-qemu-100-2026_10_29-21_00_01.vma.zst", + "content": "backup", + "format": "vma.zst", + "ctime": 1793307601, + "size": 1200000, + "vmid": 100, + "subtype": "qemu", + }, + ], +} + + +def _storage_api( + content: dict[tuple[str, str], Any] | None = None, +) -> tuple[MagicMock, dict[tuple[str, str], MagicMock]]: + """The cluster with backup storages; *content* per (node, storage).""" + rows = content or { + ("pve1", "pbs"): PBS_CONTENT, + ("pve2", "pbs"): PBS_CONTENT, + ("pve1", "local"): LOCAL_CONTENT["pve1"], + ("pve2", "local"): LOCAL_CONTENT["pve2"], + } + api = _api(resources=BACKUP_RESOURCES) + calls: dict[tuple[str, str], MagicMock] = {} + for node in ("pve1", "pve2"): + + def storage(name: str, node: str = node) -> MagicMock: + res = calls.setdefault((node, name), MagicMock()) + value = rows.get((node, name), []) + if isinstance(value, Exception): + res.content.get.side_effect = value + else: + res.content.get.return_value = value + return res + + api.nodes(node).storage.side_effect = storage + return api, calls + + +def _backups(api: MagicMock) -> dict[tuple[int, str], dict[str, Any]] | None: + backups = _report(api)["backups"] + return None if backups is None else {(b["vmid"], b["storage"]): b for b in backups} + + +class TestBackups: + def test_newest_backup_per_guest_and_storage(self): + api, _ = _storage_api() + backups = _backups(api) + assert backups is not None + assert backups[(100, "pbs")] == { + "vmid": 100, + "storage": "pbs", + "time": 1793408400, + "size": 32 * GIB, + "count": 2, + } + assert backups[(101, "pbs")]["count"] == 1 + + def test_shared_storage_read_once_on_the_own_node(self): + api, calls = _storage_api() + _report(api) + calls[("pve1", "pbs")].content.get.assert_called_once_with(content="backup") + assert ("pve2", "pbs") not in calls + + def test_local_storage_read_on_each_node_that_has_it(self): + api, calls = _storage_api() + backups = _backups(api) + assert backups is not None + # The same guest backed up to "local" on two nodes: counted together. + assert backups[(100, "local")] == { + "vmid": 100, + "storage": "local", + "time": 1793394002, + "size": 1234567, + "count": 2, + } + assert ("pve1", "local") in calls and ("pve2", "local") in calls + + def test_storage_without_backup_content_is_not_read(self): + api, calls = _storage_api() + _report(api) + assert not [key for key in calls if key[1] in ("ceph-vm", "local-lvm", "nfs-iso")] + + def test_entry_without_vmid_is_skipped(self): + api, _ = _storage_api() + backups = _backups(api) + assert backups is not None + assert {vmid for vmid, _ in backups} == {100, 101} + + def test_shared_storage_tried_on_the_next_node(self): + api, _ = _storage_api( + { + ("pve1", "pbs"): RuntimeError("pbs: datastore not reachable from pve1"), + ("pve2", "pbs"): PBS_CONTENT, + } + ) + backups = _backups(api) + assert backups is not None + assert backups[(100, "pbs")]["count"] == 2 + + def test_unreadable_storage_leaves_backups_unknown(self): + """Half a list would say a guest has no backup when it may have one.""" + api, _ = _storage_api( + { + ("pve1", "pbs"): PBS_CONTENT, + ("pve1", "local"): LOCAL_CONTENT["pve1"], + ("pve2", "local"): RuntimeError("596 Connection timed out"), + } + ) + report = _report(api) + assert report["backups"] is None + assert report["storages"] + + def test_no_backup_storage_is_an_empty_list(self): + assert _report(_api())["backups"] == [] diff --git a/tests/test_clusters.py b/tests/test_clusters.py index 985da25..72b95ea 100644 --- a/tests/test_clusters.py +++ b/tests/test_clusters.py @@ -167,6 +167,13 @@ SDN_VNETS = [ {"vnet": "overlay", "zone": "fabric", "tag": 100000, "type": "vnet", "digest": "x"}, ] +# /cluster/config/nodes: corosync's nodelist; corosync.conf keeps every value a string. +CONFIG_NODES = [ + {"node": "pve1", "name": "pve1", "nodeid": "1", "quorum_votes": "1", "ring0_addr": "10.0.0.1"}, + {"node": "pve2", "name": "pve2", "nodeid": "2", "quorum_votes": "1", "ring0_addr": "10.0.0.2"}, + {"node": "pve3", "name": "pve3", "nodeid": "3", "quorum_votes": "2", "ring0_addr": "10.0.0.3"}, +] + HA_MANAGER_STATUS = { "manager_status": { "master_node": "pve1", @@ -213,6 +220,12 @@ def _api( manager.side_effect = ha else: manager.return_value = HA_MANAGER_STATUS if ha is None else ha + # Read for get_clusters' votes, HA resources and backups; see test_cluster_ha_backups. + api.cluster.config.nodes.get.return_value = CONFIG_NODES + api.cluster.ha.resources.get.return_value = [] + api.cluster.ha.status.current.get.return_value = [] + api.cluster.ha.rules.get.return_value = [] + api.cluster.backup.get.return_value = [] return api @@ -268,6 +281,7 @@ class TestNodes: "cpu_usage": 0.25, "memory_total": 128 * GIB, "memory_used": 64 * GIB, + "votes": 1, } def test_other_member_is_not_local(self): diff --git a/tests/test_vm_migration.py b/tests/test_vm_migration.py new file mode 100644 index 0000000..542fac0 --- /dev/null +++ b/tests/test_vm_migration.py @@ -0,0 +1,382 @@ +"""Moving a guest to another node: preflight, migration, HA maintenance mode. + +Payloads: ``GET /nodes/{node}/qemu/{vmid}/migrate`` follows qemu-server's +``migrate_vm_precondition`` (PVE 9: ``not_allowed_nodes`` may name +``unavailable_storages``, ``unavailable-resources`` and +``blocking-ha-resources``); the task log follows ``/nodes/{node}/tasks/{upid}/log`` +(``n``/``t`` rows). A guest that HA manages migrates through a ``hamigrate`` +task that only hands the request to the HA manager. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from tests.test_clusters import CLUSTER_RESOURCES, CLUSTER_STATUS, GIB, _api, _driver, _storage_row + +QEMU_LIST = [ + {"vmid": 100, "name": "web", "status": "running", "maxmem": 4 * GIB}, + {"vmid": 102, "name": "idle", "status": "stopped", "maxmem": 2 * GIB}, +] +LXC_LIST = [{"vmid": "101", "name": "dns", "status": "running", "type": "lxc"}] + +PREFLIGHT_RUNNING = { + "running": 1, + "allowed_nodes": ["pve2", "pve3"], + "not_allowed_nodes": {}, + "local_disks": [ + { + "volid": "local-lvm:vm-100-disk-0", + "size": 32 * GIB, + "drivename": "scsi0", + "is_attached": 1, + "referenced_in_config": 1, + }, + {"volid": "local:iso/debian-13.iso", "cdrom": 1, "size": 700 * 1024**2}, + # Regenerated on the target: neither a disk to copy nor a blocker. + {"volid": "local-lvm:vm-100-cloudinit", "cdrom": 1, "size": 4 * 1024**2}, + ], + "local_resources": [], + "mapped-resources": [], + "mapped-resource-info": {}, + "has-dbus-vmstate": 1, +} + +PREFLIGHT_BLOCKED = { + "running": 0, + "allowed_nodes": [], + "not_allowed_nodes": { + "pve2": {"unavailable_storages": ["local-zfs"]}, + "pve3": { + "blocking-ha-resources": [{"sid": "vm:101", "cause": "resource-affinity"}], + "unavailable-resources": ["gpu-a"], + }, + }, + "local_disks": [], + "local_resources": ["hostpci0", "usb0"], + "mapped-resources": ["gpu-a"], + "mapped-resource-info": {"gpu-a": {"live-migration": 0}}, + "has-dbus-vmstate": 1, +} + +LXC_CONFIG = { + "hostname": "dns", + "rootfs": "local-lvm:vm-101-disk-0,size=8G", + "mp0": "ceph-vm:vm-101-disk-1,mp=/data,size=32G", + "mp1": "/srv/share,mp=/share", + "mp2": "/mnt/nfs,mp=/nfs,shared=1", + "unused0": "local-lvm:vm-101-disk-2", + "net0": "name=eth0,bridge=vmbr0,ip=dhcp", + "digest": "abc", +} + +UPID_QEMU = "UPID:pve1:000ABCDE:0123ABCD:672AB000:qmigrate:100:root@pam:" +UPID_LXC = "UPID:pve1:000ABCDF:0123ABCE:672AB001:vzmigrate:101:root@pam:" +UPID_HA = "UPID:pve1:000ABCE0:0123ABCF:672AB002:hamigrate:100:root@pam:" + +FAILED_LOG = [ + {"n": 1, "t": "2026-11-01 10:00:00 starting migration of VM 100 to node 'pve2' (10.0.0.2)"}, + {"n": 2, "t": "2026-11-01 10:00:01 starting VM 100 on remote node 'pve2'"}, + { + "n": 3, + "t": "2026-11-01 10:00:02 ERROR: online migrate failure - remote command failed " + "with exit code 255", + }, + {"n": 4, "t": "2026-11-01 10:00:02 aborting phase 2 - cleanup resources"}, + { + "n": 5, + "t": "2026-11-01 10:00:03 ERROR: migration finished with problems (duration 00:00:03)", + }, + {"n": 6, "t": "TASK ERROR: migration problems"}, +] + + +def _migration_api( + preflight: dict[str, Any] | None = None, + task: dict[str, Any] | None = None, + log: list[dict[str, Any]] | None = None, + status: list[dict[str, Any]] | None = None, +) -> tuple[MagicMock, MagicMock]: + resources = [ + *CLUSTER_RESOURCES, + _storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB), + ] + api = _api(cluster_status=status, resources=resources) + node = api.nodes("pve1") + node.qemu.get.return_value = QEMU_LIST + node.lxc.get.return_value = LXC_LIST + node.qemu.return_value.migrate.get.return_value = preflight or PREFLIGHT_RUNNING + node.qemu.return_value.migrate.post.return_value = UPID_QEMU + node.lxc.return_value.migrate.post.return_value = UPID_LXC + node.lxc.return_value.status.current.get.return_value = {"status": "running", "vmid": 101} + node.lxc.return_value.config.get.return_value = LXC_CONFIG + node.tasks.return_value.status.get.return_value = task or { + "status": "stopped", + "exitstatus": "OK", + } + node.tasks.return_value.log.get.return_value = log or [] + return api, node + + +class TestPreflightQemu: + def test_allowed_nodes_and_local_disks(self): + api, node = _migration_api() + result = _driver(api).get_vm_migration_preflight("web") + assert result == { + "running": True, + # pve3 is offline: the platform would allow it, it cannot take the VM now. + "allowed_nodes": ["pve2"], + "refused_nodes": {"pve3": "node offline"}, + "local_disks": ["local-lvm:vm-100-disk-0"], + # A local ISO in a CD drive stops the migration. + "local_resources": ["cdrom=local:iso/debian-13.iso"], + } + node.qemu.assert_any_call(100) + + def test_refusal_reasons_in_plain_words(self): + all_online = [dict(s, online=1) if s["type"] == "node" else s for s in CLUSTER_STATUS] + api, _ = _migration_api(preflight=PREFLIGHT_BLOCKED, status=all_online) + result = _driver(api).get_vm_migration_preflight("100") + assert result["running"] is False + assert result["allowed_nodes"] == [] + assert result["refused_nodes"] == { + "pve2": "storage local-zfs is not available there", + "pve3": "HA resource vm:101 blocks it (resource-affinity rule); " + "mapped resource gpu-a is not available there", + } + assert result["local_resources"] == ["hostpci0", "usb0"] + + def test_unknown_reason_is_passed_on(self): + preflight = { + "running": 0, + "allowed_nodes": [], + "not_allowed_nodes": {"pve2": {"some-future-check": ["x", "y"]}}, + "local_disks": [], + "local_resources": [], + } + api, _ = _migration_api(preflight=preflight) + refused = _driver(api).get_vm_migration_preflight("web")["refused_nodes"] + assert refused["pve2"] == "some-future-check: x, y" + + def test_unknown_guest(self): + api, _ = _migration_api() + with pytest.raises(ValueError, match="nope"): + _driver(api).get_vm_migration_preflight("nope") + + +class TestPreflightLxc: + def test_members_and_local_volumes(self): + api, node = _migration_api() + result = _driver(api).get_vm_migration_preflight("dns") + assert result == { + "running": True, + "allowed_nodes": ["pve2"], + "refused_nodes": {"pve3": "node offline"}, + # ceph-vm is shared; local-lvm is not. + "local_disks": ["local-lvm:vm-101-disk-0", "local-lvm:vm-101-disk-2"], + # A bind mount that is not marked shared stops the migration. + "local_resources": ["mp1"], + } + node.qemu.return_value.migrate.get.assert_not_called() + + def test_stopped_container(self): + api, node = _migration_api() + node.lxc.return_value.status.current.get.return_value = {"status": "stopped"} + assert _driver(api).get_vm_migration_preflight("101")["running"] is False + + +class TestMigrateQemu: + def test_running_vm_moves_online_with_its_local_disks(self): + api, node = _migration_api() + _driver(api).migrate_vm("web", "pve2") + node.qemu.return_value.migrate.post.assert_called_once_with( + target="pve2", online=1, **{"with-local-disks": 1} + ) + node.tasks.assert_called_with(UPID_QEMU) + + def test_target_storage_and_offline(self): + preflight = dict(PREFLIGHT_RUNNING, local_disks=[]) + api, node = _migration_api(preflight=preflight) + _driver(api).migrate_vm("web", "pve2", online=False, target_storage="ceph-vm") + node.qemu.return_value.migrate.post.assert_called_once_with( + target="pve2", targetstorage="ceph-vm" + ) + + def test_stopped_vm_is_not_moved_online(self): + preflight = dict(PREFLIGHT_RUNNING, running=0, local_disks=[]) + api, node = _migration_api(preflight=preflight) + _driver(api).migrate_vm("idle", "pve2") + node.qemu.return_value.migrate.post.assert_called_once_with(target="pve2") + + def test_failure_raises_with_the_task_logs_errors(self): + api, node = _migration_api( + task={"status": "stopped", "exitstatus": "migration problems"}, log=FAILED_LOG + ) + with pytest.raises(RuntimeError) as exc: + _driver(api).migrate_vm("web", "pve2") + message = str(exc.value) + assert "online migrate failure - remote command failed with exit code 255" in message + assert "migration finished with problems (duration 00:00:03)" in message + assert "starting VM 100" not in message + node.tasks.return_value.log.get.assert_called_once() + + def test_failure_without_error_lines_keeps_the_task_status(self): + api, _ = _migration_api( + task={"status": "stopped", "exitstatus": "migration aborted"}, + log=[{"n": 1, "t": "starting migration"}], + ) + with pytest.raises(RuntimeError, match="migration aborted"): + _driver(api).migrate_vm("web", "pve2") + + def test_refused_request_raises(self): + api, node = _migration_api() + node.qemu.return_value.migrate.post.side_effect = RuntimeError( + "500 Internal Server Error: VM is locked (backup)" + ) + with pytest.raises(RuntimeError, match="VM is locked"): + _driver(api).migrate_vm("web", "pve2") + + +class TestMigrateLxc: + def test_running_container_moves_in_restart_mode(self): + api, node = _migration_api() + _driver(api).migrate_vm("dns", "pve2", target_storage="local-lvm") + node.lxc.return_value.migrate.post.assert_called_once_with( + target="pve2", restart=1, **{"target-storage": "local-lvm"} + ) + node.tasks.assert_called_with(UPID_LXC) + + def test_stopped_container(self): + api, node = _migration_api() + node.lxc.return_value.status.current.get.return_value = {"status": "stopped"} + _driver(api).migrate_vm("dns", "pve2") + node.lxc.return_value.migrate.post.assert_called_once_with(target="pve2") + + +class TestTargetRefusedBeforeTheApi: + @pytest.mark.parametrize("target", ["pve1", "pve9", "pve2; reboot"]) + def test_not_another_member(self, target): + api, node = _migration_api() + with pytest.raises(ValueError, match="not another member"): + _driver(api).migrate_vm("web", target) + node.qemu.return_value.migrate.post.assert_not_called() + + +def _ha_status(node: str, crm_state: str) -> list[dict[str, Any]]: + return [ + {"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"}, + { + "id": "service:vm:100", + "type": "service", + "sid": "vm:100", + "node": node, + "crm_state": crm_state, + "state": crm_state, + "status": f"vm:100 ({node}, {crm_state})", + "request_state": "started", + }, + ] + + +class TestMigrateHaManaged: + """PVE hands an HA guest's migration to the HA manager: the hamigrate task only + queues the request, so the driver follows the HA manager until the guest is there.""" + + def _api(self, states: list[list[dict[str, Any]]]) -> tuple[MagicMock, MagicMock]: + api, node = _migration_api() + node.qemu.return_value.migrate.post.return_value = UPID_HA + api.cluster.ha.status.current.get.side_effect = states + return api, node + + def test_waits_until_the_guest_runs_on_the_target(self): + api, _ = self._api( + [ + _ha_status("pve1", "started"), + _ha_status("pve1", "migrate"), + _ha_status("pve2", "started"), + ] + ) + with patch("napalm_proxmox.vm_migrate_mixin.sleep") as sleep: + _driver(api).migrate_vm("web", "pve2") + assert api.cluster.ha.status.current.get.call_count == 3 + assert sleep.call_count == 2 + + def test_back_on_the_source_node_is_a_failure_with_the_migration_log(self): + api, node = self._api([_ha_status("pve1", "migrate"), _ha_status("pve1", "started")]) + node.tasks.get.return_value = [ + { + "upid": UPID_QEMU, + "type": "qmigrate", + "id": "100", + "starttime": 0x672AB002 + 12, + "status": "migration problems", + } + ] + node.tasks.return_value.log.get.return_value = FAILED_LOG + with patch("napalm_proxmox.vm_migrate_mixin.sleep"): + with pytest.raises(RuntimeError, match="online migrate failure"): + _driver(api).migrate_vm("web", "pve2") + node.tasks.get.assert_called_once_with(vmid=100, typefilter="qmigrate", limit=1) + + def test_error_state_raises(self): + api, node = self._api([_ha_status("pve1", "error")]) + node.tasks.get.return_value = [] + with patch("napalm_proxmox.vm_migrate_mixin.sleep"): + with pytest.raises(RuntimeError, match="error"): + _driver(api).migrate_vm("web", "pve2") + + def test_times_out(self): + api, _ = self._api([_ha_status("pve1", "migrate")] * 50) + clock = iter(range(0, 10000, 30)) + with ( + patch("napalm_proxmox.vm_migrate_mixin.sleep"), + patch("napalm_proxmox.vm_migrate_mixin.monotonic", lambda: next(clock)), + ): + with pytest.raises(RuntimeError, match="timed out"): + _driver(api).migrate_vm("web", "pve2", timeout=120) + + +class TestNodeMaintenance: + def _driver(self) -> tuple[Any, MagicMock]: + api, _ = _migration_api() + drv = _driver(api) + drv._node_ssh = MagicMock() # type: ignore[method-assign] + drv._run_node_command = MagicMock(return_value="") # type: ignore[method-assign] + return drv, drv._run_node_command + + def test_enable(self): + drv, run = self._driver() + drv.set_node_maintenance("pve2", True) + run.assert_called_once_with( + "ha-manager crm-command node-maintenance enable pve2", timeout=drv.timeout + ) + + def test_disable(self): + drv, run = self._driver() + drv.set_node_maintenance("pve3", False) + run.assert_called_once_with( + "ha-manager crm-command node-maintenance disable pve3", timeout=drv.timeout + ) + + @pytest.mark.parametrize("node", ["pve9", "pve2; reboot", "$(reboot)"]) + def test_not_a_member_runs_nothing(self, node): + drv, run = self._driver() + with pytest.raises(ValueError, match="not a member"): + drv.set_node_maintenance(node, True) + run.assert_not_called() + + def test_no_ssh(self): + drv, run = self._driver() + drv._node_ssh.side_effect = OSError("Authentication failed") + with pytest.raises(RuntimeError, match="SSH"): + drv.set_node_maintenance("pve2", True) + run.assert_not_called() + + def test_command_refused(self): + drv, run = self._driver() + run.side_effect = RuntimeError("Command failed (exit 255): ...\nno such node 'pve2'") + with pytest.raises(RuntimeError, match="no such node"): + drv.set_node_maintenance("pve2", True)