driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool members, the newest backup per guest on each backup storage (PBS included) and /cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a guest of the connected node and follows the task (HA-managed guests through the HA manager), raising with the task log's ERROR lines; get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance() runs ha-manager over SSH. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
556 lines
18 KiB
Python
556 lines
18 KiB
Python
"""get_clusters(): quorum votes, the HA manager's guests, backup jobs and backups.
|
|
|
|
Each of these is read on top of the cluster report and is best-effort: a read
|
|
that fails leaves its key None and the rest of the report stands.
|
|
|
|
Payloads: ``/cluster/config/nodes`` is corosync's nodelist; ``/cluster/ha/*``
|
|
follow pve-ha-manager's API (``src/PVE/API2/HA/{Resources,Status,Rules}.pm``,
|
|
master = PVE 9 and stable-4 = PVE 8): a service entry of ``status/current``
|
|
carries ``crm_state`` and ``request_state``, a rule carries ``rule``, ``type``
|
|
and ``resources`` as a sorted, comma-separated list of sids, and PVE 8 has no
|
|
``/cluster/ha/rules``. ``/cluster/backup`` follows pve-manager's
|
|
``PVE/API2/Backup.pm``; storage content rows follow
|
|
``/nodes/{node}/storage/{storage}/content``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock
|
|
|
|
from tests.test_clusters import (
|
|
CLUSTER_RESOURCES,
|
|
GIB,
|
|
_api,
|
|
_by_name,
|
|
_driver,
|
|
_storage_row,
|
|
)
|
|
|
|
|
|
def _report(api: MagicMock) -> dict[str, Any]:
|
|
[cluster] = _driver(api).get_clusters()
|
|
return dict(cluster)
|
|
|
|
|
|
class TestVotes:
|
|
def test_each_node_has_its_quorum_votes(self):
|
|
nodes = _by_name(_report(_api())["nodes"])
|
|
assert [nodes[n]["votes"] for n in ("pve1", "pve2", "pve3")] == [1, 1, 2]
|
|
|
|
def test_unreadable_nodelist_leaves_votes_unknown(self):
|
|
api = _api()
|
|
api.cluster.config.nodes.get.side_effect = RuntimeError("403 Forbidden")
|
|
nodes = _report(api)["nodes"]
|
|
assert [n["votes"] for n in nodes] == [None, None, None]
|
|
# The rest of the node still stands.
|
|
assert nodes[0]["cpu_threads"] == 32
|
|
|
|
def test_standalone_node_has_no_votes(self):
|
|
# A node without corosync.conf answers with an empty nodelist.
|
|
api = _api()
|
|
api.cluster.config.nodes.get.return_value = []
|
|
assert [n["votes"] for n in _report(api)["nodes"]] == [None, None, None]
|
|
|
|
def test_node_without_quorum_votes_counts_the_default(self):
|
|
"""corosync counts a node without quorum_votes as one vote."""
|
|
api = _api()
|
|
api.cluster.config.nodes.get.return_value = [{"node": "pve1", "name": "pve1"}]
|
|
nodes = _by_name(_report(api)["nodes"])
|
|
assert nodes["pve1"]["votes"] == 1
|
|
assert nodes["pve2"]["votes"] is None
|
|
|
|
|
|
# --------------------------------------------------------------------------- HA
|
|
|
|
HA_RESOURCES_PVE8 = [
|
|
{
|
|
"sid": "vm:100",
|
|
"type": "vm",
|
|
"state": "started",
|
|
"group": "prefer-pve1",
|
|
"max_restart": 1,
|
|
"max_relocate": 1,
|
|
"digest": "3f1c",
|
|
},
|
|
# No state in the config: HA's default, "started".
|
|
{"sid": "ct:101", "type": "ct", "digest": "3f1c"},
|
|
{"sid": "vm:102", "type": "vm", "state": "disabled", "digest": "3f1c"},
|
|
]
|
|
|
|
HA_STATUS = [
|
|
{"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"},
|
|
{
|
|
"id": "master",
|
|
"type": "master",
|
|
"node": "pve1",
|
|
"status": "pve1 (active, Sun Nov 1 10:00:00 2026)",
|
|
"timestamp": 1793527200,
|
|
},
|
|
{
|
|
"id": "lrm:pve1",
|
|
"type": "lrm",
|
|
"node": "pve1",
|
|
"status": "pve1 (active, Sun Nov 1 10:00:00 2026)",
|
|
"timestamp": 1793527200,
|
|
},
|
|
{
|
|
"id": "service:vm:100",
|
|
"type": "service",
|
|
"sid": "vm:100",
|
|
"node": "pve2",
|
|
"crm_state": "migrate",
|
|
"state": "migrate",
|
|
"status": "vm:100 (pve2, migrate)",
|
|
"request_state": "started",
|
|
"group": "prefer-pve1",
|
|
"max_restart": 1,
|
|
"max_relocate": 1,
|
|
},
|
|
{
|
|
"id": "service:ct:101",
|
|
"type": "service",
|
|
"sid": "ct:101",
|
|
"node": "pve1",
|
|
"crm_state": "started",
|
|
"state": "started",
|
|
"status": "ct:101 (pve1, started)",
|
|
"request_state": "started",
|
|
},
|
|
# Not yet processed by the CRM: no crm_state, a verbose state only.
|
|
{
|
|
"id": "service:vm:102",
|
|
"type": "service",
|
|
"sid": "vm:102",
|
|
"state": "queued",
|
|
"status": "vm:102 (---, queued)",
|
|
"request_state": "disabled",
|
|
},
|
|
]
|
|
|
|
# PVE 9: groups were migrated into node-affinity rules, so resources lose "group".
|
|
HA_RESOURCES_PVE9 = [{k: v for k, v in r.items() if k != "group"} for r in HA_RESOURCES_PVE8]
|
|
|
|
HA_RULES_PVE9 = [
|
|
{
|
|
"rule": "ha-group-prefer-pve1",
|
|
"type": "node-affinity",
|
|
"resources": "vm:100",
|
|
"nodes": "pve1:2,pve2:1",
|
|
"strict": 0,
|
|
"digest": "9ac2",
|
|
"order": 1,
|
|
},
|
|
{
|
|
"rule": "keep-apart",
|
|
"type": "resource-affinity",
|
|
"affinity": "negative",
|
|
"resources": "ct:101,vm:100",
|
|
"digest": "9ac2",
|
|
"order": 2,
|
|
},
|
|
]
|
|
|
|
|
|
def _ha_api(
|
|
resources: Any = HA_RESOURCES_PVE8,
|
|
status: Any = HA_STATUS,
|
|
rules: Any = None,
|
|
) -> MagicMock:
|
|
api = _api()
|
|
ha = api.cluster.ha
|
|
for endpoint, value in ((ha.resources, resources), (ha.status.current, status)):
|
|
if isinstance(value, Exception):
|
|
endpoint.get.side_effect = value
|
|
else:
|
|
endpoint.get.return_value = value
|
|
if rules is None:
|
|
# PVE 8 has no rules endpoint.
|
|
ha.rules.get.side_effect = RuntimeError(
|
|
"501 Not Implemented: Method 'GET /cluster/ha/rules' not implemented"
|
|
)
|
|
else:
|
|
ha.rules.get.return_value = rules
|
|
return api
|
|
|
|
|
|
def _ha(api: MagicMock) -> dict[int, dict[str, Any]] | None:
|
|
resources = _report(api)["ha_resources"]
|
|
return None if resources is None else {r["vmid"]: r for r in resources}
|
|
|
|
|
|
class TestHAResources:
|
|
def test_pve8_resource_with_group(self):
|
|
ha = _ha(_ha_api())
|
|
assert ha is not None
|
|
assert ha[100] == {
|
|
"vmid": 100,
|
|
"kind": "qemu",
|
|
"requested": "started",
|
|
"status": "migrate",
|
|
"node": "pve2",
|
|
"group": "prefer-pve1",
|
|
"rules": [],
|
|
}
|
|
|
|
def test_container_and_default_requested_state(self):
|
|
ha = _ha(_ha_api())
|
|
assert ha is not None
|
|
assert ha[101]["kind"] == "lxc"
|
|
assert ha[101]["requested"] == "started"
|
|
assert ha[101]["status"] == "started"
|
|
assert ha[101]["node"] == "pve1"
|
|
assert ha[101]["group"] is None
|
|
|
|
def test_not_yet_processed_reports_the_verbose_state(self):
|
|
ha = _ha(_ha_api())
|
|
assert ha is not None
|
|
assert ha[102]["requested"] == "disabled"
|
|
assert ha[102]["status"] == "queued"
|
|
assert ha[102]["node"] is None
|
|
|
|
def test_pve9_rules_name_their_resources(self):
|
|
ha = _ha(_ha_api(resources=HA_RESOURCES_PVE9, rules=HA_RULES_PVE9))
|
|
assert ha is not None
|
|
assert ha[100]["rules"] == ["ha-group-prefer-pve1", "keep-apart"]
|
|
assert ha[101]["rules"] == ["keep-apart"]
|
|
assert ha[102]["rules"] == []
|
|
assert ha[100]["group"] is None
|
|
|
|
def test_no_ha_configured_is_an_empty_list(self):
|
|
assert _report(_ha_api(resources=[], status=[]))["ha_resources"] == []
|
|
|
|
def test_unreadable_resources_leave_the_list_unknown(self):
|
|
api = _ha_api(resources=RuntimeError("403 Forbidden: Permission check failed"))
|
|
report = _report(api)
|
|
assert report["ha_resources"] is None
|
|
assert report["name"] == "lab"
|
|
|
|
def test_unreadable_status_leaves_status_and_node_unknown(self):
|
|
ha = _ha(_ha_api(status=RuntimeError("500 cfs lock")))
|
|
assert ha is not None
|
|
assert (ha[100]["status"], ha[100]["node"]) == (None, None)
|
|
assert ha[100]["requested"] == "started"
|
|
|
|
|
|
# ---------------------------------------------------------------------- backups
|
|
|
|
BACKUP_JOBS = [
|
|
{
|
|
"id": "backup-5a1b2c3d-e4f5",
|
|
"type": "vzdump",
|
|
"enabled": 1,
|
|
"schedule": "sat 02:00",
|
|
"storage": "pbs",
|
|
"mode": "snapshot",
|
|
"all": 1,
|
|
"exclude": "105,106",
|
|
"compress": "zstd",
|
|
"mailnotification": "always",
|
|
"next-run": 1793584800,
|
|
"notes-template": "{{guestname}}",
|
|
},
|
|
{
|
|
"id": "backup-7e8f9a0b-1c2d",
|
|
"type": "vzdump",
|
|
"enabled": 0,
|
|
"schedule": "mon..fri 21:00",
|
|
"storage": "local",
|
|
"mode": "stop",
|
|
"vmid": "100,101",
|
|
"node": "pve1",
|
|
},
|
|
# No "enabled": a job is enabled unless it says otherwise.
|
|
{
|
|
"id": "backup-0b1c2d3e-4f5a",
|
|
"type": "vzdump",
|
|
"schedule": "daily",
|
|
"storage": "pbs",
|
|
"mode": "suspend",
|
|
"pool": "prod",
|
|
},
|
|
]
|
|
|
|
# PVE >= 8.1: GET /pools?poolid=prod
|
|
POOL_INDEX = [
|
|
{
|
|
"poolid": "prod",
|
|
"comment": "",
|
|
"members": [
|
|
{"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "name": "web"},
|
|
{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "pve1", "name": "dns"},
|
|
{"id": "storage/pve1/local", "type": "storage", "storage": "local", "node": "pve1"},
|
|
],
|
|
}
|
|
]
|
|
|
|
|
|
def _backup_api(jobs: Any = BACKUP_JOBS) -> MagicMock:
|
|
api = _api()
|
|
if isinstance(jobs, Exception):
|
|
api.cluster.backup.get.side_effect = jobs
|
|
else:
|
|
api.cluster.backup.get.return_value = jobs
|
|
api.pools.get.return_value = POOL_INDEX
|
|
return api
|
|
|
|
|
|
def _jobs(api: MagicMock) -> dict[str, dict[str, Any]]:
|
|
jobs = _report(api)["backup_jobs"]
|
|
assert jobs is not None
|
|
return {j["id"]: j for j in jobs}
|
|
|
|
|
|
class TestBackupJobs:
|
|
def test_all_guests_job(self):
|
|
assert _jobs(_backup_api())["backup-5a1b2c3d-e4f5"] == {
|
|
"id": "backup-5a1b2c3d-e4f5",
|
|
"enabled": True,
|
|
"schedule": "sat 02:00",
|
|
"storage": "pbs",
|
|
"mode": "snapshot",
|
|
"all": True,
|
|
"vmids": [],
|
|
"exclude": [105, 106],
|
|
"pool": None,
|
|
"node": None,
|
|
}
|
|
|
|
def test_disabled_job_for_named_guests_on_one_node(self):
|
|
job = _jobs(_backup_api())["backup-7e8f9a0b-1c2d"]
|
|
assert job["enabled"] is False
|
|
assert job["all"] is False
|
|
assert job["vmids"] == [100, 101]
|
|
assert job["exclude"] == []
|
|
assert job["node"] == "pve1"
|
|
|
|
def test_pool_job_lists_the_pools_guests(self):
|
|
api = _backup_api()
|
|
job = _jobs(api)["backup-0b1c2d3e-4f5a"]
|
|
assert job["enabled"] is True
|
|
assert job["pool"] == "prod"
|
|
assert job["vmids"] == [100, 101]
|
|
api.pools.get.assert_called_with(poolid="prod")
|
|
|
|
def test_pool_read_falls_back_to_the_old_endpoint(self):
|
|
"""Before PVE 8.1, /pools takes no poolid; /pools/{poolid} answers instead."""
|
|
api = _backup_api()
|
|
api.pools.get.side_effect = RuntimeError("400 Bad Request: Parameter verification failed")
|
|
pool = MagicMock()
|
|
pool.get.return_value = {"comment": "", "members": POOL_INDEX[0]["members"]}
|
|
api.pools.side_effect = lambda name: pool if name == "prod" else MagicMock()
|
|
assert _jobs(api)["backup-0b1c2d3e-4f5a"]["vmids"] == [100, 101]
|
|
|
|
def test_old_style_start_time_and_days(self):
|
|
"""Jobs from vzdump.cron carry starttime/dow; PVE turns them into a schedule."""
|
|
jobs = [
|
|
{"id": "a", "starttime": "02:30", "dow": "sat,sun", "storage": "local", "vmid": "102"},
|
|
{
|
|
"id": "b",
|
|
"starttime": "03:00",
|
|
"dow": "mon,tue,wed,thu,fri,sat,sun",
|
|
"storage": "local",
|
|
"all": 1,
|
|
},
|
|
{"id": "c", "starttime": "04:00", "storage": "local", "all": 1},
|
|
]
|
|
found = _jobs(_backup_api(jobs))
|
|
assert [found[i]["schedule"] for i in "abc"] == ["sat,sun 02:30", "03:00", "04:00"]
|
|
|
|
def test_job_with_a_dump_directory_has_no_storage(self):
|
|
jobs = [{"id": "d", "schedule": "daily", "dumpdir": "/mnt/backup", "all": 1}]
|
|
job = _jobs(_backup_api(jobs))["d"]
|
|
assert job["storage"] is None
|
|
assert job["mode"] is None
|
|
|
|
def test_no_jobs_is_an_empty_list(self):
|
|
assert _report(_backup_api([]))["backup_jobs"] == []
|
|
|
|
def test_unreadable_jobs_leave_the_list_unknown(self):
|
|
report = _report(_backup_api(RuntimeError("403 Forbidden")))
|
|
assert report["backup_jobs"] is None
|
|
assert report["nodes"]
|
|
|
|
def test_unreadable_pool_leaves_the_list_unknown(self):
|
|
"""A pool job whose guests cannot be read would look like a job for none."""
|
|
api = _backup_api()
|
|
api.pools.get.side_effect = RuntimeError("403 Forbidden")
|
|
api.pools.return_value.get.side_effect = RuntimeError("403 Forbidden")
|
|
api.pools.side_effect = None
|
|
assert _report(api)["backup_jobs"] is None
|
|
|
|
|
|
BACKUP_RESOURCES = [
|
|
*CLUSTER_RESOURCES,
|
|
_storage_row("pve1", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB),
|
|
_storage_row("pve2", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB),
|
|
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB),
|
|
_storage_row("pve2", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 30 * GIB),
|
|
_storage_row("pve3", "local", "dir", 0, "iso,vztmpl,backup", 0, 0, status="unknown"),
|
|
]
|
|
|
|
PBS_CONTENT = [
|
|
{
|
|
"volid": "pbs:backup/vm/100/2026-10-31T01:00:00Z",
|
|
"content": "backup",
|
|
"format": "pbs-vm",
|
|
"ctime": 1793408400,
|
|
"size": 32 * GIB,
|
|
"vmid": 100,
|
|
"subtype": "qemu",
|
|
"notes": "web",
|
|
"protected": 0,
|
|
"verification": {"state": "ok", "upid": "UPID:pbs:0000:0000:0000:verify:x:root@pam:"},
|
|
},
|
|
{
|
|
"volid": "pbs:backup/vm/100/2026-10-24T01:00:00Z",
|
|
"content": "backup",
|
|
"format": "pbs-vm",
|
|
"ctime": 1792803600,
|
|
"size": 31 * GIB,
|
|
"vmid": 100,
|
|
"subtype": "qemu",
|
|
},
|
|
{
|
|
"volid": "pbs:backup/ct/101/2026-10-31T01:05:00Z",
|
|
"content": "backup",
|
|
"format": "pbs-ct",
|
|
"ctime": 1793408700,
|
|
"size": 2 * GIB,
|
|
"vmid": 101,
|
|
"subtype": "lxc",
|
|
},
|
|
]
|
|
|
|
LOCAL_CONTENT = {
|
|
"pve1": [
|
|
{
|
|
"volid": "local:backup/vzdump-qemu-100-2026_10_30-21_00_02.vma.zst",
|
|
"content": "backup",
|
|
"format": "vma.zst",
|
|
"ctime": 1793394002,
|
|
"size": 1234567,
|
|
"vmid": 100,
|
|
"subtype": "qemu",
|
|
},
|
|
# A file that does not follow vzdump's naming has no vmid.
|
|
{"volid": "local:backup/manual.tar", "content": "backup", "format": "tar", "size": 10},
|
|
],
|
|
"pve2": [
|
|
{
|
|
"volid": "local:backup/vzdump-qemu-100-2026_10_29-21_00_01.vma.zst",
|
|
"content": "backup",
|
|
"format": "vma.zst",
|
|
"ctime": 1793307601,
|
|
"size": 1200000,
|
|
"vmid": 100,
|
|
"subtype": "qemu",
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def _storage_api(
|
|
content: dict[tuple[str, str], Any] | None = None,
|
|
) -> tuple[MagicMock, dict[tuple[str, str], MagicMock]]:
|
|
"""The cluster with backup storages; *content* per (node, storage)."""
|
|
rows = content or {
|
|
("pve1", "pbs"): PBS_CONTENT,
|
|
("pve2", "pbs"): PBS_CONTENT,
|
|
("pve1", "local"): LOCAL_CONTENT["pve1"],
|
|
("pve2", "local"): LOCAL_CONTENT["pve2"],
|
|
}
|
|
api = _api(resources=BACKUP_RESOURCES)
|
|
calls: dict[tuple[str, str], MagicMock] = {}
|
|
for node in ("pve1", "pve2"):
|
|
|
|
def storage(name: str, node: str = node) -> MagicMock:
|
|
res = calls.setdefault((node, name), MagicMock())
|
|
value = rows.get((node, name), [])
|
|
if isinstance(value, Exception):
|
|
res.content.get.side_effect = value
|
|
else:
|
|
res.content.get.return_value = value
|
|
return res
|
|
|
|
api.nodes(node).storage.side_effect = storage
|
|
return api, calls
|
|
|
|
|
|
def _backups(api: MagicMock) -> dict[tuple[int, str], dict[str, Any]] | None:
|
|
backups = _report(api)["backups"]
|
|
return None if backups is None else {(b["vmid"], b["storage"]): b for b in backups}
|
|
|
|
|
|
class TestBackups:
|
|
def test_newest_backup_per_guest_and_storage(self):
|
|
api, _ = _storage_api()
|
|
backups = _backups(api)
|
|
assert backups is not None
|
|
assert backups[(100, "pbs")] == {
|
|
"vmid": 100,
|
|
"storage": "pbs",
|
|
"time": 1793408400,
|
|
"size": 32 * GIB,
|
|
"count": 2,
|
|
}
|
|
assert backups[(101, "pbs")]["count"] == 1
|
|
|
|
def test_shared_storage_read_once_on_the_own_node(self):
|
|
api, calls = _storage_api()
|
|
_report(api)
|
|
calls[("pve1", "pbs")].content.get.assert_called_once_with(content="backup")
|
|
assert ("pve2", "pbs") not in calls
|
|
|
|
def test_local_storage_read_on_each_node_that_has_it(self):
|
|
api, calls = _storage_api()
|
|
backups = _backups(api)
|
|
assert backups is not None
|
|
# The same guest backed up to "local" on two nodes: counted together.
|
|
assert backups[(100, "local")] == {
|
|
"vmid": 100,
|
|
"storage": "local",
|
|
"time": 1793394002,
|
|
"size": 1234567,
|
|
"count": 2,
|
|
}
|
|
assert ("pve1", "local") in calls and ("pve2", "local") in calls
|
|
|
|
def test_storage_without_backup_content_is_not_read(self):
|
|
api, calls = _storage_api()
|
|
_report(api)
|
|
assert not [key for key in calls if key[1] in ("ceph-vm", "local-lvm", "nfs-iso")]
|
|
|
|
def test_entry_without_vmid_is_skipped(self):
|
|
api, _ = _storage_api()
|
|
backups = _backups(api)
|
|
assert backups is not None
|
|
assert {vmid for vmid, _ in backups} == {100, 101}
|
|
|
|
def test_shared_storage_tried_on_the_next_node(self):
|
|
api, _ = _storage_api(
|
|
{
|
|
("pve1", "pbs"): RuntimeError("pbs: datastore not reachable from pve1"),
|
|
("pve2", "pbs"): PBS_CONTENT,
|
|
}
|
|
)
|
|
backups = _backups(api)
|
|
assert backups is not None
|
|
assert backups[(100, "pbs")]["count"] == 2
|
|
|
|
def test_unreadable_storage_leaves_backups_unknown(self):
|
|
"""Half a list would say a guest has no backup when it may have one."""
|
|
api, _ = _storage_api(
|
|
{
|
|
("pve1", "pbs"): PBS_CONTENT,
|
|
("pve1", "local"): LOCAL_CONTENT["pve1"],
|
|
("pve2", "local"): RuntimeError("596 Connection timed out"),
|
|
}
|
|
)
|
|
report = _report(api)
|
|
assert report["backups"] is None
|
|
assert report["storages"]
|
|
|
|
def test_no_backup_storage_is_an_empty_list(self):
|
|
assert _report(_api())["backups"] == []
|