Files
napalm-proxmox/tests/test_clusters.py
T
Christian Manivong 0704d9a102
driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
feat: HA state, backups and votes in get_clusters(); migrate_vm, preflight, maintenance
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool
members, the newest backup per guest on each backup storage (PBS included) and
/cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a
guest of the connected node and follows the task (HA-managed guests through
the HA manager), raising with the task log's ERROR lines;
get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance()
runs ha-manager over SSH.

Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
2026-11-01 22:15:01 +01:00

508 lines
17 KiB
Python

"""get_clusters(): the cluster a node belongs to, its nodes, storages and networks.
A Proxmox node can read its whole cluster: ``/cluster/status`` names the members
and the quorum, ``/cluster/resources`` their capacity and every storage per
node, ``/nodes/{node}/status`` the CPU, ``/nodes/{node}/network`` the bridges.
netOrk uses it to tell where a new VM may go (NetOrk/netork#484, #1822). The
payloads below follow PVE 8.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from napalm_proxmox.driver import ProxmoxDriver
from tests.conftest import _build_mock_api
GIB = 1024**3
CLUSTER_STATUS = [
{"type": "cluster", "id": "cluster", "name": "lab", "nodes": 3, "quorate": 1, "version": 7},
{
"type": "node",
"id": "node/pve1",
"name": "pve1",
"nodeid": 1,
"ip": "10.0.0.1",
"local": 1,
"online": 1,
"level": "",
},
{
"type": "node",
"id": "node/pve2",
"name": "pve2",
"nodeid": 2,
"ip": "10.0.0.2",
"local": 0,
"online": 1,
"level": "",
},
{
"type": "node",
"id": "node/pve3",
"name": "pve3",
"nodeid": 3,
"ip": "10.0.0.3",
"local": 0,
"online": 0,
"level": "",
},
]
def _node_row(node: str, maxcpu: int, cpu: float, maxmem: int, mem: int) -> dict[str, Any]:
return {
"id": f"node/{node}",
"type": "node",
"node": node,
"status": "online",
"cpu": cpu,
"maxcpu": maxcpu,
"mem": mem,
"maxmem": maxmem,
"disk": 5 * GIB,
"maxdisk": 100 * GIB,
"uptime": 86400,
"level": "",
"cgroup-mode": 2,
}
def _storage_row(
node: str,
storage: str,
plugintype: str,
shared: int,
content: str,
maxdisk: int,
disk: int,
status: str = "available",
) -> dict[str, Any]:
return {
"id": f"storage/{node}/{storage}",
"type": "storage",
"node": node,
"storage": storage,
"plugintype": plugintype,
"shared": shared,
"content": content,
"maxdisk": maxdisk,
"disk": disk,
"status": status,
}
CLUSTER_RESOURCES = [
_node_row("pve1", 32, 0.25, 128 * GIB, 64 * GIB),
_node_row("pve2", 16, 0.5, 64 * GIB, 16 * GIB),
# An offline node keeps a row, with what PVE last knew of it.
{"id": "node/pve3", "type": "node", "node": "pve3", "status": "offline", "maxcpu": 8},
_storage_row("pve1", "local-lvm", "lvmthin", 0, "rootdir,images", 400 * GIB, 100 * GIB),
_storage_row("pve2", "local-lvm", "lvmthin", 0, "rootdir,images", 200 * GIB, 50 * GIB),
_storage_row("pve3", "local-lvm", "lvmthin", 0, "rootdir,images", 0, 0, status="unknown"),
_storage_row("pve1", "ceph-vm", "rbd", 1, "images,rootdir", 3000 * GIB, 1000 * GIB),
_storage_row("pve2", "ceph-vm", "rbd", 1, "images,rootdir", 3000 * GIB, 1000 * GIB),
_storage_row("pve1", "nfs-iso", "nfs", 1, "iso,vztmpl,snippets", 1000 * GIB, 10 * GIB),
# The NFS server cannot be reached from pve2.
_storage_row("pve2", "nfs-iso", "nfs", 1, "iso,vztmpl,snippets", 0, 0, status="unavailable"),
{"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "status": "running"},
{"id": "sdn/pve1/localnetwork", "type": "sdn", "sdn": "localnetwork", "node": "pve1"},
]
def _cpuinfo(cores: int, sockets: int, model: str, flags: str) -> dict[str, Any]:
return {
"cpuinfo": {
"cores": cores,
"sockets": sockets,
"cpus": cores * sockets * 2,
"model": model,
"flags": flags,
"mhz": "2100.000",
"hvm": "1",
"user_hz": 100,
},
"memory": {"total": 128 * GIB, "used": 64 * GIB, "free": 64 * GIB},
"uptime": 86400,
"pveversion": "pve-manager/8.2.4/faa83925c9641325",
}
NODE_STATUS = {
"pve1": _cpuinfo(8, 2, "Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz", "fpu vme sse4_2 avx2"),
"pve2": _cpuinfo(8, 1, "AMD EPYC 7302P 16-Core Processor", "fpu vme sse4_2"),
}
NODE_NETWORK = {
"pve1": [
{"iface": "eno1", "type": "eth", "active": 1},
{
"iface": "vmbr0",
"type": "bridge",
"active": 1,
"bridge_ports": "eno1",
"bridge_vlan_aware": 1,
},
{"iface": "vmbr9", "type": "bridge", "active": 1, "bridge_ports": "none"},
],
"pve2": [
{"iface": "eno1", "type": "eth", "active": 1},
# Same name, but not VLAN aware on this node.
{"iface": "vmbr0", "type": "bridge", "active": 1, "bridge_ports": "eno1"},
{"iface": "vmbr1", "type": "OVSBridge", "active": 1, "ovs_ports": ""},
],
}
SDN_ZONES = [
{"zone": "edge", "type": "vlan", "bridge": "vmbr0", "nodes": "pve2", "digest": "x"},
{"zone": "fabric", "type": "vxlan", "peers": "10.0.0.1,10.0.0.2", "digest": "x"},
]
SDN_VNETS = [
{"vnet": "dmz", "zone": "edge", "tag": 30, "type": "vnet", "digest": "x"},
{"vnet": "overlay", "zone": "fabric", "tag": 100000, "type": "vnet", "digest": "x"},
]
# /cluster/config/nodes: corosync's nodelist; corosync.conf keeps every value a string.
CONFIG_NODES = [
{"node": "pve1", "name": "pve1", "nodeid": "1", "quorum_votes": "1", "ring0_addr": "10.0.0.1"},
{"node": "pve2", "name": "pve2", "nodeid": "2", "quorum_votes": "1", "ring0_addr": "10.0.0.2"},
{"node": "pve3", "name": "pve3", "nodeid": "3", "quorum_votes": "2", "ring0_addr": "10.0.0.3"},
]
HA_MANAGER_STATUS = {
"manager_status": {
"master_node": "pve1",
"node_status": {"pve1": "online", "pve2": "maintenance", "pve3": "unknown"},
"service_status": {},
"timestamp": 1730000000,
},
"quorum": {"node": "pve1", "quorate": "1"},
}
def _api(
cluster_status: list[dict[str, Any]] | None = None,
resources: list[dict[str, Any]] | None = None,
node_status: dict[str, Any] | None = None,
node_network: dict[str, Any] | None = None,
zones: list[dict[str, Any]] | None = None,
vnets: list[dict[str, Any]] | None = None,
ha: dict[str, Any] | Exception | None = None,
) -> MagicMock:
api = _build_mock_api(cluster_status=cluster_status or CLUSTER_STATUS)
statuses = NODE_STATUS if node_status is None else node_status
networks = NODE_NETWORK if node_network is None else node_network
nodes: dict[str, MagicMock] = {}
def node_res(name: str) -> MagicMock:
if name not in nodes:
res = MagicMock()
if name in statuses:
res.status.get.return_value = statuses[name]
else:
res.status.get.side_effect = ConnectionError(f"{name} unreachable")
res.network.get.return_value = networks.get(name, [])
nodes[name] = res
return nodes[name]
api.nodes.side_effect = node_res
api.nodes.return_value = None
api.cluster.resources.get.return_value = CLUSTER_RESOURCES if resources is None else resources
api.cluster.sdn.zones.get.return_value = SDN_ZONES if zones is None else zones
api.cluster.sdn.vnets.get.return_value = SDN_VNETS if vnets is None else vnets
manager = api.cluster.ha.status.manager_status.get
if isinstance(ha, Exception):
manager.side_effect = ha
else:
manager.return_value = HA_MANAGER_STATUS if ha is None else ha
# Read for get_clusters' votes, HA resources and backups; see test_cluster_ha_backups.
api.cluster.config.nodes.get.return_value = CONFIG_NODES
api.cluster.ha.resources.get.return_value = []
api.cluster.ha.status.current.get.return_value = []
api.cluster.ha.rules.get.return_value = []
api.cluster.backup.get.return_value = []
return api
def _driver(api: MagicMock, node: str = "pve1") -> ProxmoxDriver:
drv = ProxmoxDriver(
hostname=f"{node}.example.com",
username="root",
password="secret",
optional_args={"verify_ssl": False, "node": node},
)
with patch("napalm_proxmox.driver.ProxmoxAPI", return_value=api):
drv.open()
return drv
def _cluster(**kwargs: Any) -> dict[str, Any]:
clusters = _driver(_api(**kwargs)).get_clusters()
assert len(clusters) == 1
return dict(clusters[0])
def _by_name(items: list[Any]) -> dict[str, Any]:
return {item["name"]: item for item in items}
class TestCluster:
def test_name_and_quorum(self):
cluster = _cluster()
assert cluster["id"] is None
assert cluster["name"] == "lab"
assert cluster["quorate"] is True
def test_lost_quorum(self):
status = [dict(CLUSTER_STATUS[0], quorate=0), *CLUSTER_STATUS[1:]]
assert _cluster(cluster_status=status)["quorate"] is False
def test_every_member_listed_in_cluster_order(self):
assert [n["name"] for n in _cluster()["nodes"]] == ["pve1", "pve2", "pve3"]
class TestNodes:
def test_online_node(self):
pve1 = _by_name(_cluster()["nodes"])["pve1"]
assert pve1 == {
"name": "pve1",
"status": "online",
"local": True,
"maintenance": False,
"cpu_threads": 32,
"cpu_cores": 16,
"cpu_model": "Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz",
"cpu_flags": ["fpu", "vme", "sse4_2", "avx2"],
"cpu_usage": 0.25,
"memory_total": 128 * GIB,
"memory_used": 64 * GIB,
"votes": 1,
}
def test_other_member_is_not_local(self):
pve2 = _by_name(_cluster()["nodes"])["pve2"]
assert pve2["local"] is False
assert pve2["cpu_cores"] == 8
assert pve2["cpu_threads"] == 16
def test_offline_node_is_listed_without_capacity(self):
api = _api()
pve3 = _by_name(_driver(api).get_clusters()[0]["nodes"])["pve3"]
assert pve3["status"] == "offline"
assert pve3["local"] is False
for key in (
"cpu_threads",
"cpu_cores",
"cpu_model",
"cpu_flags",
"cpu_usage",
"memory_total",
"memory_used",
):
assert pve3[key] is None, key
# An offline node is not asked: the request would only time out.
assert all(call.args != ("pve3",) for call in api.nodes.call_args_list)
def test_local_falls_back_to_the_resolved_node(self):
status = [dict(s, local=0) if s["type"] == "node" else s for s in CLUSTER_STATUS]
nodes = _by_name(
_driver(_api(cluster_status=status), node="pve2").get_clusters()[0]["nodes"]
)
assert nodes["pve2"]["local"] is True
assert nodes["pve1"]["local"] is False
class TestStandalone:
STATUS = [
{
"type": "node",
"id": "node/pve1",
"name": "pve1",
"nodeid": 0,
"ip": "10.0.0.1",
"local": 1,
"online": 1,
"level": "",
}
]
RESOURCES = [
_node_row("pve1", 32, 0.25, 128 * GIB, 64 * GIB),
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 5 * GIB),
]
def test_a_cluster_of_one_without_name(self):
cluster = _cluster(cluster_status=self.STATUS, resources=self.RESOURCES, ha=RuntimeError())
assert cluster["id"] is None
assert cluster["name"] is None
assert cluster["quorate"] is None
[node] = cluster["nodes"]
assert node["name"] == "pve1"
assert node["status"] == "online"
assert node["local"] is True
assert node["cpu_cores"] == 16
assert node["memory_total"] == 128 * GIB
assert _by_name(cluster["storages"])["local"]["nodes"] == ["pve1"]
class TestStorages:
def test_local_storage_grouped_over_its_nodes(self):
lvm = _by_name(_cluster()["storages"])["local-lvm"]
assert lvm == {
"name": "local-lvm",
"type": "lvmthin",
"shared": False,
"content": ["rootdir", "images"],
# pve3 is offline: its storage is "unknown", not available.
"nodes": ["pve1", "pve2"],
# Separate disks on each node: their sizes add up.
"total": 600 * GIB,
"used": 150 * GIB,
}
def test_shared_storage_counted_once(self):
ceph = _by_name(_cluster()["storages"])["ceph-vm"]
assert ceph["shared"] is True
assert ceph["type"] == "rbd"
assert ceph["nodes"] == ["pve1", "pve2"]
assert ceph["total"] == 3000 * GIB
assert ceph["used"] == 1000 * GIB
def test_storage_unavailable_on_a_node_leaves_it_out(self):
nfs = _by_name(_cluster()["storages"])["nfs-iso"]
assert nfs["nodes"] == ["pve1"]
assert nfs["content"] == ["iso", "vztmpl", "snippets"]
assert nfs["total"] == 1000 * GIB
def test_storage_available_nowhere_has_no_size(self):
resources = [
*CLUSTER_RESOURCES[:3],
_storage_row("pve1", "nas", "cifs", 1, "backup", 0, 0, status="unavailable"),
]
nas = _by_name(_cluster(resources=resources)["storages"])["nas"]
assert nas["nodes"] == []
assert nas["total"] is None
assert nas["used"] is None
class TestNetworks:
def test_bridge_merged_across_nodes_vlan_aware_only_if_everywhere(self):
vmbr0 = _by_name(_cluster()["networks"])["vmbr0"]
assert vmbr0 == {
"name": "vmbr0",
"kind": "bridge",
"vlan_aware": False,
"nodes": ["pve1", "pve2"],
}
def test_bridge_on_one_node(self):
networks = _by_name(_cluster()["networks"])
assert networks["vmbr9"]["nodes"] == ["pve1"]
assert networks["vmbr1"] == {
"name": "vmbr1",
"kind": "bridge",
"vlan_aware": True,
"nodes": ["pve2"],
}
def test_vlan_aware_on_every_node(self):
network = {
n: [dict(i, bridge_vlan_aware=1) for i in NODE_NETWORK["pve1"]] for n in NODE_NETWORK
}
assert _by_name(_cluster(node_network=network)["networks"])["vmbr0"]["vlan_aware"] is True
def test_vnet_on_its_zones_nodes(self):
dmz = _by_name(_cluster()["networks"])["dmz"]
assert dmz == {
"name": "dmz",
"kind": "vnet",
"vlan_aware": False,
"fixed_vlan_tag": 30,
"nodes": ["pve2"],
}
def test_vnet_of_a_zone_without_nodes_is_everywhere(self):
overlay = _by_name(_cluster()["networks"])["overlay"]
assert overlay["nodes"] == ["pve1", "pve2", "pve3"]
assert overlay["fixed_vlan_tag"] == 100000
def test_get_network_targets_unchanged_for_the_own_node(self):
targets = _by_name(_driver(_api()).get_network_targets())
assert targets["vmbr0"] == {
"name": "vmbr0",
"kind": "bridge",
"vlan_aware": True,
"uplinks": [["eno1"]],
}
assert targets["dmz"]["fixed_vlan_tag"] == 30
class TestMaintenance:
def test_ha_reports_maintenance(self):
nodes = _by_name(_cluster()["nodes"])
assert nodes["pve1"]["maintenance"] is False
assert nodes["pve2"]["maintenance"] is True
def test_ha_unreadable_leaves_maintenance_unknown(self):
nodes = _cluster(ha=RuntimeError("403 Forbidden: Permission check failed"))["nodes"]
assert [n["maintenance"] for n in nodes] == [None, None, None]
def test_node_the_ha_manager_does_not_track_is_unknown(self):
ha = {"manager_status": {"node_status": {"pve1": "online"}}, "quorum": {}}
nodes = _by_name(_cluster(ha=ha)["nodes"])
assert nodes["pve1"]["maintenance"] is False
assert nodes["pve2"]["maintenance"] is None
class TestReadFailures:
def test_cluster_resources_unreadable_raises(self):
api = _api()
api.cluster.resources.get.side_effect = ConnectionError("pveproxy down")
with pytest.raises(ConnectionError):
_driver(api).get_clusters()
def test_cluster_status_unreadable_raises(self):
api = _api()
drv = _driver(api)
api.cluster.status.get.side_effect = ConnectionError("pveproxy down")
with pytest.raises(ConnectionError):
drv.get_clusters()
def test_own_node_status_unreadable_raises(self):
status = {"pve2": NODE_STATUS["pve2"]}
with pytest.raises(ConnectionError):
_driver(_api(node_status=status)).get_clusters()
def test_other_node_status_unreadable_leaves_its_cpu_unknown(self):
status = {"pve1": NODE_STATUS["pve1"]}
pve2 = _by_name(_cluster(node_status=status)["nodes"])["pve2"]
assert pve2["status"] == "online"
assert pve2["cpu_cores"] is None
assert pve2["cpu_model"] is None
assert pve2["cpu_flags"] is None
# What /cluster/resources says still stands.
assert pve2["cpu_threads"] == 16
class TestCreateOnAnotherNode:
def test_other_node_raises_before_anything_is_created(self):
api = _api()
drv = _driver(api)
with pytest.raises(ValueError, match="pve2"):
drv.create_vm_from_cloud_init(
"vm1",
image_url="https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2",
cpu=2,
memory=2048,
nics=[{"bridge": "vmbr0"}],
cloud_init_config={},
node="pve2",
)
api.cluster.nextid.get.assert_not_called()
for res in [api.nodes(n) for n in ("pve1", "pve2")]:
assert not [c for c in res.mock_calls if "post" in c[0]]