"""get_clusters(): quorum votes, the HA manager's guests, backup jobs and backups. Each of these is read on top of the cluster report and is best-effort: a read that fails leaves its key None and the rest of the report stands. Payloads: ``/cluster/config/nodes`` is corosync's nodelist; ``/cluster/ha/*`` follow pve-ha-manager's API (``src/PVE/API2/HA/{Resources,Status,Rules}.pm``, master = PVE 9 and stable-4 = PVE 8): a service entry of ``status/current`` carries ``crm_state`` and ``request_state``, a rule carries ``rule``, ``type`` and ``resources`` as a sorted, comma-separated list of sids, and PVE 8 has no ``/cluster/ha/rules``. ``/cluster/backup`` follows pve-manager's ``PVE/API2/Backup.pm``; storage content rows follow ``/nodes/{node}/storage/{storage}/content``. """ from __future__ import annotations from typing import Any from unittest.mock import MagicMock from tests.test_clusters import ( CLUSTER_RESOURCES, GIB, _api, _by_name, _driver, _storage_row, ) def _report(api: MagicMock) -> dict[str, Any]: [cluster] = _driver(api).get_clusters() return dict(cluster) class TestVotes: def test_each_node_has_its_quorum_votes(self): nodes = _by_name(_report(_api())["nodes"]) assert [nodes[n]["votes"] for n in ("pve1", "pve2", "pve3")] == [1, 1, 2] def test_unreadable_nodelist_leaves_votes_unknown(self): api = _api() api.cluster.config.nodes.get.side_effect = RuntimeError("403 Forbidden") nodes = _report(api)["nodes"] assert [n["votes"] for n in nodes] == [None, None, None] # The rest of the node still stands. assert nodes[0]["cpu_threads"] == 32 def test_standalone_node_has_no_votes(self): # A node without corosync.conf answers with an empty nodelist. api = _api() api.cluster.config.nodes.get.return_value = [] assert [n["votes"] for n in _report(api)["nodes"]] == [None, None, None] def test_node_without_quorum_votes_counts_the_default(self): """corosync counts a node without quorum_votes as one vote.""" api = _api() api.cluster.config.nodes.get.return_value = [{"node": "pve1", "name": "pve1"}] nodes = _by_name(_report(api)["nodes"]) assert nodes["pve1"]["votes"] == 1 assert nodes["pve2"]["votes"] is None # --------------------------------------------------------------------------- HA HA_RESOURCES_PVE8 = [ { "sid": "vm:100", "type": "vm", "state": "started", "group": "prefer-pve1", "max_restart": 1, "max_relocate": 1, "digest": "3f1c", }, # No state in the config: HA's default, "started". {"sid": "ct:101", "type": "ct", "digest": "3f1c"}, {"sid": "vm:102", "type": "vm", "state": "disabled", "digest": "3f1c"}, ] HA_STATUS = [ {"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"}, { "id": "master", "type": "master", "node": "pve1", "status": "pve1 (active, Sun Nov 1 10:00:00 2026)", "timestamp": 1793527200, }, { "id": "lrm:pve1", "type": "lrm", "node": "pve1", "status": "pve1 (active, Sun Nov 1 10:00:00 2026)", "timestamp": 1793527200, }, { "id": "service:vm:100", "type": "service", "sid": "vm:100", "node": "pve2", "crm_state": "migrate", "state": "migrate", "status": "vm:100 (pve2, migrate)", "request_state": "started", "group": "prefer-pve1", "max_restart": 1, "max_relocate": 1, }, { "id": "service:ct:101", "type": "service", "sid": "ct:101", "node": "pve1", "crm_state": "started", "state": "started", "status": "ct:101 (pve1, started)", "request_state": "started", }, # Not yet processed by the CRM: no crm_state, a verbose state only. { "id": "service:vm:102", "type": "service", "sid": "vm:102", "state": "queued", "status": "vm:102 (---, queued)", "request_state": "disabled", }, ] # PVE 9: groups were migrated into node-affinity rules, so resources lose "group". HA_RESOURCES_PVE9 = [{k: v for k, v in r.items() if k != "group"} for r in HA_RESOURCES_PVE8] HA_RULES_PVE9 = [ { "rule": "ha-group-prefer-pve1", "type": "node-affinity", "resources": "vm:100", "nodes": "pve1:2,pve2:1", "strict": 0, "digest": "9ac2", "order": 1, }, { "rule": "keep-apart", "type": "resource-affinity", "affinity": "negative", "resources": "ct:101,vm:100", "digest": "9ac2", "order": 2, }, ] def _ha_api( resources: Any = HA_RESOURCES_PVE8, status: Any = HA_STATUS, rules: Any = None, ) -> MagicMock: api = _api() ha = api.cluster.ha for endpoint, value in ((ha.resources, resources), (ha.status.current, status)): if isinstance(value, Exception): endpoint.get.side_effect = value else: endpoint.get.return_value = value if rules is None: # PVE 8 has no rules endpoint. ha.rules.get.side_effect = RuntimeError( "501 Not Implemented: Method 'GET /cluster/ha/rules' not implemented" ) else: ha.rules.get.return_value = rules return api def _ha(api: MagicMock) -> dict[int, dict[str, Any]] | None: resources = _report(api)["ha_resources"] return None if resources is None else {r["vmid"]: r for r in resources} class TestHAResources: def test_pve8_resource_with_group(self): ha = _ha(_ha_api()) assert ha is not None assert ha[100] == { "vmid": 100, "kind": "qemu", "requested": "started", "status": "migrate", "node": "pve2", "group": "prefer-pve1", "rules": [], } def test_container_and_default_requested_state(self): ha = _ha(_ha_api()) assert ha is not None assert ha[101]["kind"] == "lxc" assert ha[101]["requested"] == "started" assert ha[101]["status"] == "started" assert ha[101]["node"] == "pve1" assert ha[101]["group"] is None def test_not_yet_processed_reports_the_verbose_state(self): ha = _ha(_ha_api()) assert ha is not None assert ha[102]["requested"] == "disabled" assert ha[102]["status"] == "queued" assert ha[102]["node"] is None def test_pve9_rules_name_their_resources(self): ha = _ha(_ha_api(resources=HA_RESOURCES_PVE9, rules=HA_RULES_PVE9)) assert ha is not None assert ha[100]["rules"] == ["ha-group-prefer-pve1", "keep-apart"] assert ha[101]["rules"] == ["keep-apart"] assert ha[102]["rules"] == [] assert ha[100]["group"] is None def test_no_ha_configured_is_an_empty_list(self): assert _report(_ha_api(resources=[], status=[]))["ha_resources"] == [] def test_unreadable_resources_leave_the_list_unknown(self): api = _ha_api(resources=RuntimeError("403 Forbidden: Permission check failed")) report = _report(api) assert report["ha_resources"] is None assert report["name"] == "lab" def test_unreadable_status_leaves_status_and_node_unknown(self): ha = _ha(_ha_api(status=RuntimeError("500 cfs lock"))) assert ha is not None assert (ha[100]["status"], ha[100]["node"]) == (None, None) assert ha[100]["requested"] == "started" # ---------------------------------------------------------------------- backups BACKUP_JOBS = [ { "id": "backup-5a1b2c3d-e4f5", "type": "vzdump", "enabled": 1, "schedule": "sat 02:00", "storage": "pbs", "mode": "snapshot", "all": 1, "exclude": "105,106", "compress": "zstd", "mailnotification": "always", "next-run": 1793584800, "notes-template": "{{guestname}}", }, { "id": "backup-7e8f9a0b-1c2d", "type": "vzdump", "enabled": 0, "schedule": "mon..fri 21:00", "storage": "local", "mode": "stop", "vmid": "100,101", "node": "pve1", }, # No "enabled": a job is enabled unless it says otherwise. { "id": "backup-0b1c2d3e-4f5a", "type": "vzdump", "schedule": "daily", "storage": "pbs", "mode": "suspend", "pool": "prod", }, ] # PVE >= 8.1: GET /pools?poolid=prod POOL_INDEX = [ { "poolid": "prod", "comment": "", "members": [ {"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "name": "web"}, {"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "pve1", "name": "dns"}, {"id": "storage/pve1/local", "type": "storage", "storage": "local", "node": "pve1"}, ], } ] def _backup_api(jobs: Any = BACKUP_JOBS) -> MagicMock: api = _api() if isinstance(jobs, Exception): api.cluster.backup.get.side_effect = jobs else: api.cluster.backup.get.return_value = jobs api.pools.get.return_value = POOL_INDEX return api def _jobs(api: MagicMock) -> dict[str, dict[str, Any]]: jobs = _report(api)["backup_jobs"] assert jobs is not None return {j["id"]: j for j in jobs} class TestBackupJobs: def test_all_guests_job(self): assert _jobs(_backup_api())["backup-5a1b2c3d-e4f5"] == { "id": "backup-5a1b2c3d-e4f5", "enabled": True, "schedule": "sat 02:00", "storage": "pbs", "mode": "snapshot", "all": True, "vmids": [], "exclude": [105, 106], "pool": None, "node": None, } def test_disabled_job_for_named_guests_on_one_node(self): job = _jobs(_backup_api())["backup-7e8f9a0b-1c2d"] assert job["enabled"] is False assert job["all"] is False assert job["vmids"] == [100, 101] assert job["exclude"] == [] assert job["node"] == "pve1" def test_pool_job_lists_the_pools_guests(self): api = _backup_api() job = _jobs(api)["backup-0b1c2d3e-4f5a"] assert job["enabled"] is True assert job["pool"] == "prod" assert job["vmids"] == [100, 101] api.pools.get.assert_called_with(poolid="prod") def test_pool_read_falls_back_to_the_old_endpoint(self): """Before PVE 8.1, /pools takes no poolid; /pools/{poolid} answers instead.""" api = _backup_api() api.pools.get.side_effect = RuntimeError("400 Bad Request: Parameter verification failed") pool = MagicMock() pool.get.return_value = {"comment": "", "members": POOL_INDEX[0]["members"]} api.pools.side_effect = lambda name: pool if name == "prod" else MagicMock() assert _jobs(api)["backup-0b1c2d3e-4f5a"]["vmids"] == [100, 101] def test_old_style_start_time_and_days(self): """Jobs from vzdump.cron carry starttime/dow; PVE turns them into a schedule.""" jobs = [ {"id": "a", "starttime": "02:30", "dow": "sat,sun", "storage": "local", "vmid": "102"}, { "id": "b", "starttime": "03:00", "dow": "mon,tue,wed,thu,fri,sat,sun", "storage": "local", "all": 1, }, {"id": "c", "starttime": "04:00", "storage": "local", "all": 1}, ] found = _jobs(_backup_api(jobs)) assert [found[i]["schedule"] for i in "abc"] == ["sat,sun 02:30", "03:00", "04:00"] def test_job_with_a_dump_directory_has_no_storage(self): jobs = [{"id": "d", "schedule": "daily", "dumpdir": "/mnt/backup", "all": 1}] job = _jobs(_backup_api(jobs))["d"] assert job["storage"] is None assert job["mode"] is None def test_no_jobs_is_an_empty_list(self): assert _report(_backup_api([]))["backup_jobs"] == [] def test_unreadable_jobs_leave_the_list_unknown(self): report = _report(_backup_api(RuntimeError("403 Forbidden"))) assert report["backup_jobs"] is None assert report["nodes"] def test_unreadable_pool_leaves_the_list_unknown(self): """A pool job whose guests cannot be read would look like a job for none.""" api = _backup_api() api.pools.get.side_effect = RuntimeError("403 Forbidden") api.pools.return_value.get.side_effect = RuntimeError("403 Forbidden") api.pools.side_effect = None assert _report(api)["backup_jobs"] is None BACKUP_RESOURCES = [ *CLUSTER_RESOURCES, _storage_row("pve1", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB), _storage_row("pve2", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB), _storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB), _storage_row("pve2", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 30 * GIB), _storage_row("pve3", "local", "dir", 0, "iso,vztmpl,backup", 0, 0, status="unknown"), ] PBS_CONTENT = [ { "volid": "pbs:backup/vm/100/2026-10-31T01:00:00Z", "content": "backup", "format": "pbs-vm", "ctime": 1793408400, "size": 32 * GIB, "vmid": 100, "subtype": "qemu", "notes": "web", "protected": 0, "verification": {"state": "ok", "upid": "UPID:pbs:0000:0000:0000:verify:x:root@pam:"}, }, { "volid": "pbs:backup/vm/100/2026-10-24T01:00:00Z", "content": "backup", "format": "pbs-vm", "ctime": 1792803600, "size": 31 * GIB, "vmid": 100, "subtype": "qemu", }, { "volid": "pbs:backup/ct/101/2026-10-31T01:05:00Z", "content": "backup", "format": "pbs-ct", "ctime": 1793408700, "size": 2 * GIB, "vmid": 101, "subtype": "lxc", }, ] LOCAL_CONTENT = { "pve1": [ { "volid": "local:backup/vzdump-qemu-100-2026_10_30-21_00_02.vma.zst", "content": "backup", "format": "vma.zst", "ctime": 1793394002, "size": 1234567, "vmid": 100, "subtype": "qemu", }, # A file that does not follow vzdump's naming has no vmid. {"volid": "local:backup/manual.tar", "content": "backup", "format": "tar", "size": 10}, ], "pve2": [ { "volid": "local:backup/vzdump-qemu-100-2026_10_29-21_00_01.vma.zst", "content": "backup", "format": "vma.zst", "ctime": 1793307601, "size": 1200000, "vmid": 100, "subtype": "qemu", }, ], } def _storage_api( content: dict[tuple[str, str], Any] | None = None, ) -> tuple[MagicMock, dict[tuple[str, str], MagicMock]]: """The cluster with backup storages; *content* per (node, storage).""" rows = content or { ("pve1", "pbs"): PBS_CONTENT, ("pve2", "pbs"): PBS_CONTENT, ("pve1", "local"): LOCAL_CONTENT["pve1"], ("pve2", "local"): LOCAL_CONTENT["pve2"], } api = _api(resources=BACKUP_RESOURCES) calls: dict[tuple[str, str], MagicMock] = {} for node in ("pve1", "pve2"): def storage(name: str, node: str = node) -> MagicMock: res = calls.setdefault((node, name), MagicMock()) value = rows.get((node, name), []) if isinstance(value, Exception): res.content.get.side_effect = value else: res.content.get.return_value = value return res api.nodes(node).storage.side_effect = storage return api, calls def _backups(api: MagicMock) -> dict[tuple[int, str], dict[str, Any]] | None: backups = _report(api)["backups"] return None if backups is None else {(b["vmid"], b["storage"]): b for b in backups} class TestBackups: def test_newest_backup_per_guest_and_storage(self): api, _ = _storage_api() backups = _backups(api) assert backups is not None assert backups[(100, "pbs")] == { "vmid": 100, "storage": "pbs", "time": 1793408400, "size": 32 * GIB, "count": 2, } assert backups[(101, "pbs")]["count"] == 1 def test_shared_storage_read_once_on_the_own_node(self): api, calls = _storage_api() _report(api) calls[("pve1", "pbs")].content.get.assert_called_once_with(content="backup") assert ("pve2", "pbs") not in calls def test_local_storage_read_on_each_node_that_has_it(self): api, calls = _storage_api() backups = _backups(api) assert backups is not None # The same guest backed up to "local" on two nodes: counted together. assert backups[(100, "local")] == { "vmid": 100, "storage": "local", "time": 1793394002, "size": 1234567, "count": 2, } assert ("pve1", "local") in calls and ("pve2", "local") in calls def test_storage_without_backup_content_is_not_read(self): api, calls = _storage_api() _report(api) assert not [key for key in calls if key[1] in ("ceph-vm", "local-lvm", "nfs-iso")] def test_entry_without_vmid_is_skipped(self): api, _ = _storage_api() backups = _backups(api) assert backups is not None assert {vmid for vmid, _ in backups} == {100, 101} def test_shared_storage_tried_on_the_next_node(self): api, _ = _storage_api( { ("pve1", "pbs"): RuntimeError("pbs: datastore not reachable from pve1"), ("pve2", "pbs"): PBS_CONTENT, } ) backups = _backups(api) assert backups is not None assert backups[(100, "pbs")]["count"] == 2 def test_unreadable_storage_leaves_backups_unknown(self): """Half a list would say a guest has no backup when it may have one.""" api, _ = _storage_api( { ("pve1", "pbs"): PBS_CONTENT, ("pve1", "local"): LOCAL_CONTENT["pve1"], ("pve2", "local"): RuntimeError("596 Connection timed out"), } ) report = _report(api) assert report["backups"] is None assert report["storages"] def test_no_backup_storage_is_an_empty_list(self): assert _report(_api())["backups"] == []