driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool members, the newest backup per guest on each backup storage (PBS included) and /cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a guest of the connected node and follows the task (HA-managed guests through the HA manager), raising with the task log's ERROR lines; get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance() runs ha-manager over SSH. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
508 lines
17 KiB
Python
508 lines
17 KiB
Python
"""get_clusters(): the cluster a node belongs to, its nodes, storages and networks.
|
|
|
|
A Proxmox node can read its whole cluster: ``/cluster/status`` names the members
|
|
and the quorum, ``/cluster/resources`` their capacity and every storage per
|
|
node, ``/nodes/{node}/status`` the CPU, ``/nodes/{node}/network`` the bridges.
|
|
netOrk uses it to tell where a new VM may go (NetOrk/netork#484, #1822). The
|
|
payloads below follow PVE 8.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from napalm_proxmox.driver import ProxmoxDriver
|
|
from tests.conftest import _build_mock_api
|
|
|
|
GIB = 1024**3
|
|
|
|
CLUSTER_STATUS = [
|
|
{"type": "cluster", "id": "cluster", "name": "lab", "nodes": 3, "quorate": 1, "version": 7},
|
|
{
|
|
"type": "node",
|
|
"id": "node/pve1",
|
|
"name": "pve1",
|
|
"nodeid": 1,
|
|
"ip": "10.0.0.1",
|
|
"local": 1,
|
|
"online": 1,
|
|
"level": "",
|
|
},
|
|
{
|
|
"type": "node",
|
|
"id": "node/pve2",
|
|
"name": "pve2",
|
|
"nodeid": 2,
|
|
"ip": "10.0.0.2",
|
|
"local": 0,
|
|
"online": 1,
|
|
"level": "",
|
|
},
|
|
{
|
|
"type": "node",
|
|
"id": "node/pve3",
|
|
"name": "pve3",
|
|
"nodeid": 3,
|
|
"ip": "10.0.0.3",
|
|
"local": 0,
|
|
"online": 0,
|
|
"level": "",
|
|
},
|
|
]
|
|
|
|
|
|
def _node_row(node: str, maxcpu: int, cpu: float, maxmem: int, mem: int) -> dict[str, Any]:
|
|
return {
|
|
"id": f"node/{node}",
|
|
"type": "node",
|
|
"node": node,
|
|
"status": "online",
|
|
"cpu": cpu,
|
|
"maxcpu": maxcpu,
|
|
"mem": mem,
|
|
"maxmem": maxmem,
|
|
"disk": 5 * GIB,
|
|
"maxdisk": 100 * GIB,
|
|
"uptime": 86400,
|
|
"level": "",
|
|
"cgroup-mode": 2,
|
|
}
|
|
|
|
|
|
def _storage_row(
|
|
node: str,
|
|
storage: str,
|
|
plugintype: str,
|
|
shared: int,
|
|
content: str,
|
|
maxdisk: int,
|
|
disk: int,
|
|
status: str = "available",
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"id": f"storage/{node}/{storage}",
|
|
"type": "storage",
|
|
"node": node,
|
|
"storage": storage,
|
|
"plugintype": plugintype,
|
|
"shared": shared,
|
|
"content": content,
|
|
"maxdisk": maxdisk,
|
|
"disk": disk,
|
|
"status": status,
|
|
}
|
|
|
|
|
|
CLUSTER_RESOURCES = [
|
|
_node_row("pve1", 32, 0.25, 128 * GIB, 64 * GIB),
|
|
_node_row("pve2", 16, 0.5, 64 * GIB, 16 * GIB),
|
|
# An offline node keeps a row, with what PVE last knew of it.
|
|
{"id": "node/pve3", "type": "node", "node": "pve3", "status": "offline", "maxcpu": 8},
|
|
_storage_row("pve1", "local-lvm", "lvmthin", 0, "rootdir,images", 400 * GIB, 100 * GIB),
|
|
_storage_row("pve2", "local-lvm", "lvmthin", 0, "rootdir,images", 200 * GIB, 50 * GIB),
|
|
_storage_row("pve3", "local-lvm", "lvmthin", 0, "rootdir,images", 0, 0, status="unknown"),
|
|
_storage_row("pve1", "ceph-vm", "rbd", 1, "images,rootdir", 3000 * GIB, 1000 * GIB),
|
|
_storage_row("pve2", "ceph-vm", "rbd", 1, "images,rootdir", 3000 * GIB, 1000 * GIB),
|
|
_storage_row("pve1", "nfs-iso", "nfs", 1, "iso,vztmpl,snippets", 1000 * GIB, 10 * GIB),
|
|
# The NFS server cannot be reached from pve2.
|
|
_storage_row("pve2", "nfs-iso", "nfs", 1, "iso,vztmpl,snippets", 0, 0, status="unavailable"),
|
|
{"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "status": "running"},
|
|
{"id": "sdn/pve1/localnetwork", "type": "sdn", "sdn": "localnetwork", "node": "pve1"},
|
|
]
|
|
|
|
|
|
def _cpuinfo(cores: int, sockets: int, model: str, flags: str) -> dict[str, Any]:
|
|
return {
|
|
"cpuinfo": {
|
|
"cores": cores,
|
|
"sockets": sockets,
|
|
"cpus": cores * sockets * 2,
|
|
"model": model,
|
|
"flags": flags,
|
|
"mhz": "2100.000",
|
|
"hvm": "1",
|
|
"user_hz": 100,
|
|
},
|
|
"memory": {"total": 128 * GIB, "used": 64 * GIB, "free": 64 * GIB},
|
|
"uptime": 86400,
|
|
"pveversion": "pve-manager/8.2.4/faa83925c9641325",
|
|
}
|
|
|
|
|
|
NODE_STATUS = {
|
|
"pve1": _cpuinfo(8, 2, "Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz", "fpu vme sse4_2 avx2"),
|
|
"pve2": _cpuinfo(8, 1, "AMD EPYC 7302P 16-Core Processor", "fpu vme sse4_2"),
|
|
}
|
|
|
|
NODE_NETWORK = {
|
|
"pve1": [
|
|
{"iface": "eno1", "type": "eth", "active": 1},
|
|
{
|
|
"iface": "vmbr0",
|
|
"type": "bridge",
|
|
"active": 1,
|
|
"bridge_ports": "eno1",
|
|
"bridge_vlan_aware": 1,
|
|
},
|
|
{"iface": "vmbr9", "type": "bridge", "active": 1, "bridge_ports": "none"},
|
|
],
|
|
"pve2": [
|
|
{"iface": "eno1", "type": "eth", "active": 1},
|
|
# Same name, but not VLAN aware on this node.
|
|
{"iface": "vmbr0", "type": "bridge", "active": 1, "bridge_ports": "eno1"},
|
|
{"iface": "vmbr1", "type": "OVSBridge", "active": 1, "ovs_ports": ""},
|
|
],
|
|
}
|
|
|
|
SDN_ZONES = [
|
|
{"zone": "edge", "type": "vlan", "bridge": "vmbr0", "nodes": "pve2", "digest": "x"},
|
|
{"zone": "fabric", "type": "vxlan", "peers": "10.0.0.1,10.0.0.2", "digest": "x"},
|
|
]
|
|
|
|
SDN_VNETS = [
|
|
{"vnet": "dmz", "zone": "edge", "tag": 30, "type": "vnet", "digest": "x"},
|
|
{"vnet": "overlay", "zone": "fabric", "tag": 100000, "type": "vnet", "digest": "x"},
|
|
]
|
|
|
|
# /cluster/config/nodes: corosync's nodelist; corosync.conf keeps every value a string.
|
|
CONFIG_NODES = [
|
|
{"node": "pve1", "name": "pve1", "nodeid": "1", "quorum_votes": "1", "ring0_addr": "10.0.0.1"},
|
|
{"node": "pve2", "name": "pve2", "nodeid": "2", "quorum_votes": "1", "ring0_addr": "10.0.0.2"},
|
|
{"node": "pve3", "name": "pve3", "nodeid": "3", "quorum_votes": "2", "ring0_addr": "10.0.0.3"},
|
|
]
|
|
|
|
HA_MANAGER_STATUS = {
|
|
"manager_status": {
|
|
"master_node": "pve1",
|
|
"node_status": {"pve1": "online", "pve2": "maintenance", "pve3": "unknown"},
|
|
"service_status": {},
|
|
"timestamp": 1730000000,
|
|
},
|
|
"quorum": {"node": "pve1", "quorate": "1"},
|
|
}
|
|
|
|
|
|
def _api(
|
|
cluster_status: list[dict[str, Any]] | None = None,
|
|
resources: list[dict[str, Any]] | None = None,
|
|
node_status: dict[str, Any] | None = None,
|
|
node_network: dict[str, Any] | None = None,
|
|
zones: list[dict[str, Any]] | None = None,
|
|
vnets: list[dict[str, Any]] | None = None,
|
|
ha: dict[str, Any] | Exception | None = None,
|
|
) -> MagicMock:
|
|
api = _build_mock_api(cluster_status=cluster_status or CLUSTER_STATUS)
|
|
statuses = NODE_STATUS if node_status is None else node_status
|
|
networks = NODE_NETWORK if node_network is None else node_network
|
|
nodes: dict[str, MagicMock] = {}
|
|
|
|
def node_res(name: str) -> MagicMock:
|
|
if name not in nodes:
|
|
res = MagicMock()
|
|
if name in statuses:
|
|
res.status.get.return_value = statuses[name]
|
|
else:
|
|
res.status.get.side_effect = ConnectionError(f"{name} unreachable")
|
|
res.network.get.return_value = networks.get(name, [])
|
|
nodes[name] = res
|
|
return nodes[name]
|
|
|
|
api.nodes.side_effect = node_res
|
|
api.nodes.return_value = None
|
|
api.cluster.resources.get.return_value = CLUSTER_RESOURCES if resources is None else resources
|
|
api.cluster.sdn.zones.get.return_value = SDN_ZONES if zones is None else zones
|
|
api.cluster.sdn.vnets.get.return_value = SDN_VNETS if vnets is None else vnets
|
|
manager = api.cluster.ha.status.manager_status.get
|
|
if isinstance(ha, Exception):
|
|
manager.side_effect = ha
|
|
else:
|
|
manager.return_value = HA_MANAGER_STATUS if ha is None else ha
|
|
# Read for get_clusters' votes, HA resources and backups; see test_cluster_ha_backups.
|
|
api.cluster.config.nodes.get.return_value = CONFIG_NODES
|
|
api.cluster.ha.resources.get.return_value = []
|
|
api.cluster.ha.status.current.get.return_value = []
|
|
api.cluster.ha.rules.get.return_value = []
|
|
api.cluster.backup.get.return_value = []
|
|
return api
|
|
|
|
|
|
def _driver(api: MagicMock, node: str = "pve1") -> ProxmoxDriver:
|
|
drv = ProxmoxDriver(
|
|
hostname=f"{node}.example.com",
|
|
username="root",
|
|
password="secret",
|
|
optional_args={"verify_ssl": False, "node": node},
|
|
)
|
|
with patch("napalm_proxmox.driver.ProxmoxAPI", return_value=api):
|
|
drv.open()
|
|
return drv
|
|
|
|
|
|
def _cluster(**kwargs: Any) -> dict[str, Any]:
|
|
clusters = _driver(_api(**kwargs)).get_clusters()
|
|
assert len(clusters) == 1
|
|
return dict(clusters[0])
|
|
|
|
|
|
def _by_name(items: list[Any]) -> dict[str, Any]:
|
|
return {item["name"]: item for item in items}
|
|
|
|
|
|
class TestCluster:
|
|
def test_name_and_quorum(self):
|
|
cluster = _cluster()
|
|
assert cluster["id"] is None
|
|
assert cluster["name"] == "lab"
|
|
assert cluster["quorate"] is True
|
|
|
|
def test_lost_quorum(self):
|
|
status = [dict(CLUSTER_STATUS[0], quorate=0), *CLUSTER_STATUS[1:]]
|
|
assert _cluster(cluster_status=status)["quorate"] is False
|
|
|
|
def test_every_member_listed_in_cluster_order(self):
|
|
assert [n["name"] for n in _cluster()["nodes"]] == ["pve1", "pve2", "pve3"]
|
|
|
|
|
|
class TestNodes:
|
|
def test_online_node(self):
|
|
pve1 = _by_name(_cluster()["nodes"])["pve1"]
|
|
assert pve1 == {
|
|
"name": "pve1",
|
|
"status": "online",
|
|
"local": True,
|
|
"maintenance": False,
|
|
"cpu_threads": 32,
|
|
"cpu_cores": 16,
|
|
"cpu_model": "Intel(R) Xeon(R) Silver 4110 CPU @ 2.10GHz",
|
|
"cpu_flags": ["fpu", "vme", "sse4_2", "avx2"],
|
|
"cpu_usage": 0.25,
|
|
"memory_total": 128 * GIB,
|
|
"memory_used": 64 * GIB,
|
|
"votes": 1,
|
|
}
|
|
|
|
def test_other_member_is_not_local(self):
|
|
pve2 = _by_name(_cluster()["nodes"])["pve2"]
|
|
assert pve2["local"] is False
|
|
assert pve2["cpu_cores"] == 8
|
|
assert pve2["cpu_threads"] == 16
|
|
|
|
def test_offline_node_is_listed_without_capacity(self):
|
|
api = _api()
|
|
pve3 = _by_name(_driver(api).get_clusters()[0]["nodes"])["pve3"]
|
|
assert pve3["status"] == "offline"
|
|
assert pve3["local"] is False
|
|
for key in (
|
|
"cpu_threads",
|
|
"cpu_cores",
|
|
"cpu_model",
|
|
"cpu_flags",
|
|
"cpu_usage",
|
|
"memory_total",
|
|
"memory_used",
|
|
):
|
|
assert pve3[key] is None, key
|
|
# An offline node is not asked: the request would only time out.
|
|
assert all(call.args != ("pve3",) for call in api.nodes.call_args_list)
|
|
|
|
def test_local_falls_back_to_the_resolved_node(self):
|
|
status = [dict(s, local=0) if s["type"] == "node" else s for s in CLUSTER_STATUS]
|
|
nodes = _by_name(
|
|
_driver(_api(cluster_status=status), node="pve2").get_clusters()[0]["nodes"]
|
|
)
|
|
assert nodes["pve2"]["local"] is True
|
|
assert nodes["pve1"]["local"] is False
|
|
|
|
|
|
class TestStandalone:
|
|
STATUS = [
|
|
{
|
|
"type": "node",
|
|
"id": "node/pve1",
|
|
"name": "pve1",
|
|
"nodeid": 0,
|
|
"ip": "10.0.0.1",
|
|
"local": 1,
|
|
"online": 1,
|
|
"level": "",
|
|
}
|
|
]
|
|
RESOURCES = [
|
|
_node_row("pve1", 32, 0.25, 128 * GIB, 64 * GIB),
|
|
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 5 * GIB),
|
|
]
|
|
|
|
def test_a_cluster_of_one_without_name(self):
|
|
cluster = _cluster(cluster_status=self.STATUS, resources=self.RESOURCES, ha=RuntimeError())
|
|
assert cluster["id"] is None
|
|
assert cluster["name"] is None
|
|
assert cluster["quorate"] is None
|
|
[node] = cluster["nodes"]
|
|
assert node["name"] == "pve1"
|
|
assert node["status"] == "online"
|
|
assert node["local"] is True
|
|
assert node["cpu_cores"] == 16
|
|
assert node["memory_total"] == 128 * GIB
|
|
assert _by_name(cluster["storages"])["local"]["nodes"] == ["pve1"]
|
|
|
|
|
|
class TestStorages:
|
|
def test_local_storage_grouped_over_its_nodes(self):
|
|
lvm = _by_name(_cluster()["storages"])["local-lvm"]
|
|
assert lvm == {
|
|
"name": "local-lvm",
|
|
"type": "lvmthin",
|
|
"shared": False,
|
|
"content": ["rootdir", "images"],
|
|
# pve3 is offline: its storage is "unknown", not available.
|
|
"nodes": ["pve1", "pve2"],
|
|
# Separate disks on each node: their sizes add up.
|
|
"total": 600 * GIB,
|
|
"used": 150 * GIB,
|
|
}
|
|
|
|
def test_shared_storage_counted_once(self):
|
|
ceph = _by_name(_cluster()["storages"])["ceph-vm"]
|
|
assert ceph["shared"] is True
|
|
assert ceph["type"] == "rbd"
|
|
assert ceph["nodes"] == ["pve1", "pve2"]
|
|
assert ceph["total"] == 3000 * GIB
|
|
assert ceph["used"] == 1000 * GIB
|
|
|
|
def test_storage_unavailable_on_a_node_leaves_it_out(self):
|
|
nfs = _by_name(_cluster()["storages"])["nfs-iso"]
|
|
assert nfs["nodes"] == ["pve1"]
|
|
assert nfs["content"] == ["iso", "vztmpl", "snippets"]
|
|
assert nfs["total"] == 1000 * GIB
|
|
|
|
def test_storage_available_nowhere_has_no_size(self):
|
|
resources = [
|
|
*CLUSTER_RESOURCES[:3],
|
|
_storage_row("pve1", "nas", "cifs", 1, "backup", 0, 0, status="unavailable"),
|
|
]
|
|
nas = _by_name(_cluster(resources=resources)["storages"])["nas"]
|
|
assert nas["nodes"] == []
|
|
assert nas["total"] is None
|
|
assert nas["used"] is None
|
|
|
|
|
|
class TestNetworks:
|
|
def test_bridge_merged_across_nodes_vlan_aware_only_if_everywhere(self):
|
|
vmbr0 = _by_name(_cluster()["networks"])["vmbr0"]
|
|
assert vmbr0 == {
|
|
"name": "vmbr0",
|
|
"kind": "bridge",
|
|
"vlan_aware": False,
|
|
"nodes": ["pve1", "pve2"],
|
|
}
|
|
|
|
def test_bridge_on_one_node(self):
|
|
networks = _by_name(_cluster()["networks"])
|
|
assert networks["vmbr9"]["nodes"] == ["pve1"]
|
|
assert networks["vmbr1"] == {
|
|
"name": "vmbr1",
|
|
"kind": "bridge",
|
|
"vlan_aware": True,
|
|
"nodes": ["pve2"],
|
|
}
|
|
|
|
def test_vlan_aware_on_every_node(self):
|
|
network = {
|
|
n: [dict(i, bridge_vlan_aware=1) for i in NODE_NETWORK["pve1"]] for n in NODE_NETWORK
|
|
}
|
|
assert _by_name(_cluster(node_network=network)["networks"])["vmbr0"]["vlan_aware"] is True
|
|
|
|
def test_vnet_on_its_zones_nodes(self):
|
|
dmz = _by_name(_cluster()["networks"])["dmz"]
|
|
assert dmz == {
|
|
"name": "dmz",
|
|
"kind": "vnet",
|
|
"vlan_aware": False,
|
|
"fixed_vlan_tag": 30,
|
|
"nodes": ["pve2"],
|
|
}
|
|
|
|
def test_vnet_of_a_zone_without_nodes_is_everywhere(self):
|
|
overlay = _by_name(_cluster()["networks"])["overlay"]
|
|
assert overlay["nodes"] == ["pve1", "pve2", "pve3"]
|
|
assert overlay["fixed_vlan_tag"] == 100000
|
|
|
|
def test_get_network_targets_unchanged_for_the_own_node(self):
|
|
targets = _by_name(_driver(_api()).get_network_targets())
|
|
assert targets["vmbr0"] == {
|
|
"name": "vmbr0",
|
|
"kind": "bridge",
|
|
"vlan_aware": True,
|
|
"uplinks": [["eno1"]],
|
|
}
|
|
assert targets["dmz"]["fixed_vlan_tag"] == 30
|
|
|
|
|
|
class TestMaintenance:
|
|
def test_ha_reports_maintenance(self):
|
|
nodes = _by_name(_cluster()["nodes"])
|
|
assert nodes["pve1"]["maintenance"] is False
|
|
assert nodes["pve2"]["maintenance"] is True
|
|
|
|
def test_ha_unreadable_leaves_maintenance_unknown(self):
|
|
nodes = _cluster(ha=RuntimeError("403 Forbidden: Permission check failed"))["nodes"]
|
|
assert [n["maintenance"] for n in nodes] == [None, None, None]
|
|
|
|
def test_node_the_ha_manager_does_not_track_is_unknown(self):
|
|
ha = {"manager_status": {"node_status": {"pve1": "online"}}, "quorum": {}}
|
|
nodes = _by_name(_cluster(ha=ha)["nodes"])
|
|
assert nodes["pve1"]["maintenance"] is False
|
|
assert nodes["pve2"]["maintenance"] is None
|
|
|
|
|
|
class TestReadFailures:
|
|
def test_cluster_resources_unreadable_raises(self):
|
|
api = _api()
|
|
api.cluster.resources.get.side_effect = ConnectionError("pveproxy down")
|
|
with pytest.raises(ConnectionError):
|
|
_driver(api).get_clusters()
|
|
|
|
def test_cluster_status_unreadable_raises(self):
|
|
api = _api()
|
|
drv = _driver(api)
|
|
api.cluster.status.get.side_effect = ConnectionError("pveproxy down")
|
|
with pytest.raises(ConnectionError):
|
|
drv.get_clusters()
|
|
|
|
def test_own_node_status_unreadable_raises(self):
|
|
status = {"pve2": NODE_STATUS["pve2"]}
|
|
with pytest.raises(ConnectionError):
|
|
_driver(_api(node_status=status)).get_clusters()
|
|
|
|
def test_other_node_status_unreadable_leaves_its_cpu_unknown(self):
|
|
status = {"pve1": NODE_STATUS["pve1"]}
|
|
pve2 = _by_name(_cluster(node_status=status)["nodes"])["pve2"]
|
|
assert pve2["status"] == "online"
|
|
assert pve2["cpu_cores"] is None
|
|
assert pve2["cpu_model"] is None
|
|
assert pve2["cpu_flags"] is None
|
|
# What /cluster/resources says still stands.
|
|
assert pve2["cpu_threads"] == 16
|
|
|
|
|
|
class TestCreateOnAnotherNode:
|
|
def test_other_node_raises_before_anything_is_created(self):
|
|
api = _api()
|
|
drv = _driver(api)
|
|
with pytest.raises(ValueError, match="pve2"):
|
|
drv.create_vm_from_cloud_init(
|
|
"vm1",
|
|
image_url="https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2",
|
|
cpu=2,
|
|
memory=2048,
|
|
nics=[{"bridge": "vmbr0"}],
|
|
cloud_init_config={},
|
|
node="pve2",
|
|
)
|
|
api.cluster.nextid.get.assert_not_called()
|
|
for res in [api.nodes(n) for n in ("pve1", "pve2")]:
|
|
assert not [c for c in res.mock_calls if "post" in c[0]]
|