feat: HA state, backups and votes in get_clusters(); migrate_vm, preflight, maintenance
driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped

get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool
members, the newest backup per guest on each backup storage (PBS included) and
/cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a
guest of the connected node and follows the task (HA-managed guests through
the HA manager), raising with the task log's ERROR lines;
get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance()
runs ha-manager over SSH.

Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
This commit is contained in:
Christian Manivong committed 2026-11-01 22:15:01 +01:00
1 parent cb29a42c31
commit 0704d9a102
11 files changed
+1653 -17

No files matched your search

+25
View File
@@ -12,6 +12,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6).
### Added
- `get_clusters()` reports quorum `votes` per node (`/cluster/config/nodes`), the
guests HA manages in `ha_resources` (`/cluster/ha/resources`, the CRM state and
node from `/cluster/ha/status/current`, the ids of the PVE 9 HA rules that name
each from `/cluster/ha/rules`, `[]` on PVE 8), the vzdump jobs in `backup_jobs`
(`/cluster/backup`; a pool job lists the pool's guests from `/pools?poolid=`, or
`/pools/{pool}` before PVE 8.1; old `starttime`/`dow` jobs get the schedule PVE
itself derives) and per guest and storage the newest backup in `backups` (each
storage with backup content read once if shared, on each node if local; PBS
included). Each is best-effort: one that cannot be read is None and the rest of
the report stands.
- `get_vm_migration_preflight(name)`: a VM's nodes and blockers from
`GET .../qemu/{vmid}/migrate` in plain words, a local ISO in a CD drive as a
blocking resource, the cloud-init drive left out; a container's candidates are the
online members, its local volumes and unshared bind mounts read from its config.
An offline member is refused as `"node offline"` for both.
- `migrate_vm(name, target_node, online=True, target_storage=None, timeout=3600)`:
`POST .../{qemu|lxc}/{vmid}/migrate` (`online` for a running VM, `with-local-disks`
when it has local disks, restart mode for a running container), followed to the
end; a failure raises with the task log's `ERROR:` lines. A target that is not
another cluster member is refused before the API is called. A guest HA manages is
followed through the HA manager until it is on the target.
- `set_node_maintenance(node, enabled)`: `ha-manager crm-command node-maintenance`
over the driver's SSH session, for a cluster member only and quoted; raises
RuntimeError when SSH is not available. Needs the shapes and contracts of the
next napalm-device-types release for type checking (runtime imports none).
- `get_clusters()`: the cluster the node belongs to, always exactly one, read
from `/cluster/status`, `/cluster/resources`, each online node's status and
network, SDN vnets/zones and the HA manager. Every member is listed, an
+151
View File
@@ -0,0 +1,151 @@
"""get_clusters() for Proxmox VE: the backup jobs, and the backups on the storages.
Read from:
- ``GET /cluster/backup``: each vzdump job. ``enabled`` absent means enabled;
old jobs carry ``starttime``/``dow`` instead of ``schedule``, which PVE itself
turns into ``"<dow> <starttime>"`` (``PVE/API2/Backup.pm``). ``storage`` and
``mode`` are None where the job does not set them: a node's
``/etc/vzdump.conf`` then decides.
- a pool job's guests: ``GET /pools?poolid=<pool>`` (PVE 8.1 and later), else
``GET /pools/<pool>``, which older releases have;
- ``GET /nodes/<node>/storage/<storage>/content?content=backup`` for each storage
that holds backups (PBS answers the same): a shared one once, on the own node
if it sees it, else on the next node that does; a local one on every node
that has it.
Each list is best-effort: when part of it cannot be read it is None, because
half a list would say a guest has no backup, or a job no guest, when it may.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from napalm_proxmox.pve_values import split_list, to_flag, to_int, vmid_list
if TYPE_CHECKING:
# Type-only: newer than the napalm_device_types floor in pyproject.toml.
from napalm_device_types import BackupDict, BackupJobDict, ClusterStorageDict
logger = logging.getLogger(__name__)
_JsonDict = dict[str, Any]
_ALL_DAYS = "mon,tue,wed,thu,fri,sat,sun"
_GUEST_TYPES = ("qemu", "lxc")
def _schedule(job: _JsonDict) -> str | None:
if job.get("schedule"):
return str(job["schedule"])
start = job.get("starttime")
if not start:
return None
dow = ",".join(split_list(job.get("dow")))
return str(start) if dow in ("", _ALL_DAYS) else f"{dow} {start}"
def _pool_info(api: Any, pool: str) -> _JsonDict:
try:
for entry in api.pools.get(poolid=pool) or []:
if entry.get("poolid") == pool:
return dict(entry)
except Exception as exc:
logger.debug("GET /pools?poolid=%s failed, trying /pools/%s: %s", pool, pool, exc)
return dict(api.pools(pool).get() or {})
def _pool_members(api: Any, pool: str) -> list[int]:
members = _pool_info(api, pool).get("members") or []
return [
int(m["vmid"])
for m in members
if m.get("type") in _GUEST_TYPES and m.get("vmid") is not None
]
def _job(api: Any, job: _JsonDict) -> BackupJobDict:
pool = job.get("pool") or None
return {
"id": str(job.get("id") or ""),
"enabled": to_flag(job.get("enabled", 1)),
"schedule": _schedule(job),
"storage": job.get("storage") or None,
"mode": job.get("mode") or None,
"all": to_flag(job.get("all")),
"vmids": _pool_members(api, str(pool)) if pool else vmid_list(job.get("vmid")),
"exclude": vmid_list(job.get("exclude")),
"pool": pool,
"node": job.get("node") or None,
}
def read_backup_jobs(api: Any) -> list[BackupJobDict] | None:
"""The cluster's backup jobs; None when they, or a pool's guests, cannot be read."""
try:
return [_job(api, job) for job in api.cluster.backup.get() or []]
except Exception as exc:
logger.debug("Cannot read the backup jobs: %s", exc)
return None
def _content(api: Any, node: str, storage: str) -> list[_JsonDict]:
return list(api.nodes(node).storage(storage).content.get(content="backup") or [])
def _first_answer(api: Any, storage: str, nodes: list[str]) -> list[_JsonDict]:
error: Exception | None = None
for node in nodes:
try:
return _content(api, node, storage)
except Exception as exc:
logger.debug("Cannot read backups on %s from %s: %s", storage, node, exc)
error = exc
if error is not None:
raise error
return []
def _storage_rows(api: Any, storage: ClusterStorageDict, local_node: str) -> list[_JsonDict]:
nodes = storage["nodes"]
if storage["shared"]:
return _first_answer(api, storage["name"], sorted(nodes, key=lambda n: n != local_node))
return [row for node in nodes for row in _content(api, node, storage["name"])]
def _merge(rows: list[_JsonDict], storage: str, merged: dict[tuple[int, str], BackupDict]) -> None:
for row in rows:
if row.get("vmid") is None or row.get("ctime") is None:
continue # not a vzdump/PBS backup of a guest
key, ctime = (int(row["vmid"]), storage), int(row["ctime"])
entry = merged.get(key)
if entry is None:
merged[key] = {
"vmid": key[0],
"storage": storage,
"time": ctime,
"size": to_int(row.get("size")),
"count": 1,
}
continue
entry["count"] += 1
if ctime > entry["time"]:
entry["time"], entry["size"] = ctime, to_int(row.get("size"))
def read_backups(
api: Any, storages: list[ClusterStorageDict], local_node: str
) -> list[BackupDict] | None:
"""One entry per guest and storage with its newest backup; None if a storage
that holds backups cannot be read."""
merged: dict[tuple[int, str], BackupDict] = {}
try:
for storage in storages:
if "backup" in storage["content"]:
_merge(_storage_rows(api, storage, local_node), storage["name"], merged)
except Exception as exc:
logger.debug("Cannot read the backups: %s", exc)
return None
return [merged[key] for key in sorted(merged, key=lambda k: (k[1], k[0]))]
+105
View File
@@ -0,0 +1,105 @@
"""get_clusters() for Proxmox VE: quorum votes and the guests the HA manager keeps.
Read from:
- ``GET /cluster/config/nodes``: corosync's nodelist, ``quorum_votes`` per node;
- ``GET /cluster/ha/resources``: each HA resource's ``sid`` (``vm:100``,
``ct:101``), requested ``state`` and, on PVE 8, ``group``;
- ``GET /cluster/ha/status/current``: per ``service`` entry the ``node`` and the
CRM's ``crm_state`` (only the verbose ``state`` while the CRM has not taken it
up yet);
- ``GET /cluster/ha/rules`` (PVE 9): each rule's ``rule`` id and its
``resources``, a comma-separated list of sids. PVE 8 has no such endpoint.
Each read is best-effort: what cannot be read is None, never a guess.
"""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from napalm_proxmox.pve_values import split_list
if TYPE_CHECKING:
# Type-only: newer than the napalm_device_types floor in pyproject.toml.
from napalm_device_types import HAResourceDict
logger = logging.getLogger(__name__)
_JsonDict = dict[str, Any]
#: HA resource type prefix -> guest kind.
_KINDS = {"vm": "qemu", "ct": "lxc"}
#: corosync counts a node without ``quorum_votes`` as one vote.
_DEFAULT_VOTES = 1
#: What HA keeps a resource at when its config names no state.
_DEFAULT_REQUESTED = "started"
def read_votes(api: Any) -> dict[str, int] | None:
"""Quorum votes per node name; None when the nodelist cannot be read."""
try:
rows = api.cluster.config.nodes.get() or []
except Exception as exc:
logger.debug("Cannot read the corosync nodelist: %s", exc)
return None
return {
str(row.get("node") or row.get("name")): int(row.get("quorum_votes", _DEFAULT_VOTES))
for row in rows
if row.get("node") or row.get("name")
}
def _services(api: Any) -> dict[str, _JsonDict] | None:
try:
rows = api.cluster.ha.status.current.get() or []
except Exception as exc:
logger.debug("Cannot read the HA status: %s", exc)
return None
return {str(r["sid"]): r for r in rows if r.get("type") == "service" and r.get("sid")}
def _rules(api: Any) -> dict[str, list[str]]:
"""Rule ids per sid. PVE 8 answers with an error: no rules there."""
try:
rows = api.cluster.ha.rules.get() or []
except Exception as exc:
logger.debug("No HA rules (PVE 8 has none): %s", exc)
return {}
by_sid: dict[str, list[str]] = {}
for rule in rows:
for sid in split_list(rule.get("resources")):
by_sid.setdefault(sid, []).append(str(rule.get("rule")))
return by_sid
def _resource(
row: _JsonDict, services: dict[str, _JsonDict] | None, rules: dict[str, list[str]]
) -> HAResourceDict | None:
sid = str(row.get("sid") or "")
prefix, _, vmid = sid.partition(":")
if prefix not in _KINDS or not vmid.isdigit():
return None
service = (services or {}).get(sid, {})
return {
"vmid": int(vmid),
"kind": _KINDS[prefix],
"requested": str(row.get("state") or _DEFAULT_REQUESTED),
"status": service.get("crm_state") or service.get("state") or None,
"node": service.get("node") or None,
"group": row.get("group") or None,
"rules": rules.get(sid, []),
}
def read_ha_resources(api: Any) -> list[HAResourceDict] | None:
"""Every guest HA manages, by vmid; None when the HA resources cannot be read."""
try:
rows = api.cluster.ha.resources.get() or []
except Exception as exc:
logger.debug("Cannot read the HA resources: %s", exc)
return None
services, rules = _services(api), _rules(api)
resources = [_resource(row, services, rules) for row in rows]
return sorted((r for r in resources if r is not None), key=lambda r: r["vmid"])
+22 -17
View File
@@ -13,7 +13,10 @@ cluster of one, without a name). Read from:
- ``GET /nodes/{node}/network`` of each online node: its bridges;
- ``GET /cluster/sdn/vnets`` and ``/cluster/sdn/zones``: the vnets, on the nodes
of their zone;
- ``GET /cluster/ha/status/manager_status``: which node is in maintenance.
- ``GET /cluster/ha/status/manager_status``: which node is in maintenance;
- quorum votes and HA resources: see ``cluster_ha``; backup jobs and backups:
see ``cluster_backups``. Each of those is best-effort and None when it cannot
be read, without failing the rest of the report.
"""
from __future__ import annotations
@@ -21,7 +24,12 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, NamedTuple
from napalm_proxmox.cluster_backups import read_backup_jobs, read_backups
from napalm_proxmox.cluster_ha import read_ha_resources, read_votes
from napalm_proxmox.network_targets import bridge_targets, vnet_target
from napalm_proxmox.pve_values import split_list as _list
from napalm_proxmox.pve_values import to_flag as _flag
from napalm_proxmox.pve_values import to_int as _int
from napalm_proxmox.utils import cpu_flags
if TYPE_CHECKING:
@@ -51,22 +59,10 @@ class _Cpu(NamedTuple):
flags: list[str] | None = None
def _int(value: Any) -> int | None:
return None if value is None else int(value)
def _float(value: Any) -> float | None:
return None if value is None else float(value)
def _flag(value: Any) -> bool:
return bool(int(value or 0))
def _list(value: Any) -> list[str]:
return [part.strip() for part in str(value or "").split(",") if part.strip()]
def _size(rows: list[_JsonDict], key: str, shared: bool) -> int | None:
"""One storage's size over the nodes in *rows*: summed unless it is shared."""
values = [row.get(key) for row in rows]
@@ -151,14 +147,18 @@ class ProxmoxClusterMixin:
resources = self._api.cluster.resources.get() or []
head = next((s for s in status if s.get("type") == "cluster"), None)
members = self._members(status if head else [])
storages = _storages(resources)
return [
{
"id": None,
"name": (head.get("name") or None) if head else None,
"quorate": _flag(head.get("quorate")) if head else None,
"nodes": self._cluster_nodes(members, resources),
"storages": _storages(resources),
"storages": storages,
"networks": self._networks(members),
"ha_resources": read_ha_resources(self._api),
"backup_jobs": read_backup_jobs(self._api),
"backups": read_backups(self._api, storages, self._node_name),
}
]
@@ -166,8 +166,8 @@ class ProxmoxClusterMixin:
self, members: list[_Member], resources: list[_JsonDict]
) -> list[ClusterNodeDict]:
rows = {r.get("node"): r for r in resources if r.get("type") == "node"}
maintenance = self._maintenance()
return [self._cluster_node(m, rows.get(m.name, {}), maintenance) for m in members]
maintenance, votes = self._maintenance(), read_votes(self._api)
return [self._cluster_node(m, rows.get(m.name, {}), maintenance, votes) for m in members]
def _networks(self, members: list[_Member]) -> list[ClusterNetworkDict]:
online = [m for m in members if m.online]
@@ -225,7 +225,11 @@ class ProxmoxClusterMixin:
)
def _cluster_node(
self, member: _Member, row: _JsonDict, maintenance: dict[str, bool] | None
self,
member: _Member,
row: _JsonDict,
maintenance: dict[str, bool] | None,
votes: dict[str, int] | None,
) -> ClusterNodeDict:
if not member.online:
row, cpu = {}, _Cpu()
@@ -243,6 +247,7 @@ class ProxmoxClusterMixin:
"cpu_usage": _float(row.get("cpu")),
"memory_total": _int(row.get("maxmem")),
"memory_used": _int(row.get("mem")),
"votes": None if votes is None else votes.get(member.name),
}
def _vnets(self, members: list[str]) -> list[ClusterNetworkDict]:
+2
View File
@@ -63,6 +63,7 @@ from napalm_proxmox.routing_mixin import ProxmoxRoutingMixin
from napalm_proxmox.sdn_mixin import ProxmoxSDNMixin
from napalm_proxmox.system_mixin import ProxmoxSystemMixin
from napalm_proxmox.vm_contract_mixin import ProxmoxVMContractMixin
from napalm_proxmox.vm_migrate_mixin import ProxmoxVMMigrateMixin
from napalm_proxmox.vm_mixin import ProxmoxVMMixin
from napalm_proxmox.vm_provision_mixin import ProxmoxVMProvisionMixin
from napalm_proxmox.vm_snapshot_mixin import ProxmoxVMSnapshotMixin
@@ -89,6 +90,7 @@ class ProxmoxDriver(
ProxmoxVMSnapshotMixin,
ProxmoxVMProvisionMixin,
ProxmoxClusterMixin,
ProxmoxVMMigrateMixin,
ProxmoxRoutingMixin,
ProxmoxSystemMixin,
KernelFactsMixin,
+115
View File
@@ -0,0 +1,115 @@
"""Reading Proxmox VE's answers about a guest's migration: refusal reasons, local
volumes and resources, task log errors.
- ``GET /nodes/{node}/qemu/{vmid}/migrate`` (qemu-server's
``migrate_vm_precondition``): ``not_allowed_nodes`` maps a node to its
blockers -- ``unavailable_storages``, ``unavailable-resources`` (mappings the
node lacks), ``blocking-ha-resources`` (PVE 9 HA rules); ``local_disks`` lists
volumes on non-shared storage, CD drives included; ``local_resources``
(``hostpci0``, ``usb0``, ``clipboard=vnc``) block the migration.
- A container has no such endpoint; its config says what it keeps locally
(``PVE/LXC/Migrate.pm``): a bind or device mount point that is not marked
``shared`` stops the migration, a volume on non-shared storage is copied.
- A task log is rows of ``{"n": <line>, "t": <text>}``; a failure writes
``ERROR: ...`` lines.
"""
from __future__ import annotations
import re
from typing import Any
from napalm_proxmox.pve_values import to_flag
_JsonDict = dict[str, Any]
_ERROR = re.compile(r"\bERROR: (.*\S)")
#: The cloud-init drive is a CD drive the migration regenerates on the target.
_CLOUDINIT = re.compile(r"vm-\d+-cloudinit")
_CT_VOLUME_KEY = re.compile(r"^(rootfs|mp\d+|unused\d+)$")
def task_errors(log: list[_JsonDict]) -> list[str]:
"""The ``ERROR:`` lines of a task log, without their timestamps, each once."""
errors: list[str] = []
for row in log:
match = _ERROR.search(str(row.get("t") or ""))
if match and match.group(1) not in errors:
errors.append(match.group(1))
return errors
def upid_field(upid: str, index: int) -> str:
"""A field of ``UPID:node:pid:pstart:starttime:type:id:user:``; "" if absent."""
parts = upid.split(":")
return parts[index] if len(parts) > index else ""
def _names(kind: str, items: list[Any]) -> str:
names = ", ".join(str(item) for item in items)
return f"{kind} {names} is" if len(items) == 1 else f"{kind}s {names} are"
def _ha_block(item: Any) -> str:
if not isinstance(item, dict):
return f"HA resource {item} blocks it"
cause = f" ({item['cause']} rule)" if item.get("cause") else ""
return f"HA resource {item.get('sid')} blocks it{cause}"
def _reason(key: str, value: Any) -> str:
items = value if isinstance(value, list) else [value]
if key == "unavailable_storages":
return f"{_names('storage', items)} not available there"
if key == "unavailable-resources":
return f"{_names('mapped resource', items)} not available there"
if key == "blocking-ha-resources":
return "; ".join(_ha_block(item) for item in items)
return f"{key}: {', '.join(str(item) for item in items)}"
def refusal(blockers: Any) -> str:
"""One node's ``not_allowed_nodes`` entry in plain words."""
if not isinstance(blockers, dict) or not blockers:
return "refused by Proxmox VE"
return "; ".join(_reason(key, blockers[key]) for key in sorted(blockers))
def qemu_local(answer: _JsonDict) -> tuple[list[str], list[str]]:
"""``(local_disks, local_resources)`` of a VM's migrate precondition answer.
A local ISO in a CD drive stops the migration, so it is a resource
(``cdrom=<volid>``, after PVE's own ``clipboard=vnc``), not a disk.
"""
disks: list[str] = []
resources = [str(r) for r in answer.get("local_resources") or []]
for disk in answer.get("local_disks") or []:
volid = str(disk.get("volid") or "")
if not volid or _CLOUDINIT.search(volid):
continue
if to_flag(disk.get("cdrom")):
resources.append(f"cdrom={volid}")
else:
disks.append(volid)
return sorted(disks), resources
def lxc_local(config: _JsonDict, shared: dict[str, bool]) -> tuple[list[str], list[str]]:
"""``(local_disks, local_resources)`` of a container's config.
*shared* says per storage whether it is shared; a storage it does not know is
taken as local, which blocks nothing.
"""
disks: list[str] = []
resources: list[str] = []
for key in sorted(config):
if not _CT_VOLUME_KEY.match(key):
continue
volume, *options = str(config[key]).split(",")
volume = volume.removeprefix("volume=")
if volume.startswith("/"):
if "shared=1" not in options:
resources.append(key)
elif not shared.get(volume.split(":", 1)[0], False):
disks.append(volume)
return sorted(disks), resources
+27
View File
@@ -0,0 +1,27 @@
"""How the Proxmox VE API writes plain values: numbers as strings, flags as 0/1, lists
as comma-separated strings."""
from __future__ import annotations
import re
from typing import Any
_LIST_SEPARATORS = re.compile(r"[\s,;\0]+")
def to_int(value: Any) -> int | None:
return None if value is None else int(value)
def to_flag(value: Any) -> bool:
return bool(int(value or 0))
def split_list(value: Any) -> list[str]:
"""``"a,b"`` -> ``["a", "b"]``; PVE also separates with ``;``, blanks or NUL."""
return [part for part in _LIST_SEPARATORS.split(str(value or "")) if part]
def vmid_list(value: Any) -> list[int]:
"""``"100,101"`` -> ``[100, 101]``; what is not a number is left out."""
return [int(part) for part in split_list(value) if part.isdigit()]
+255
View File
@@ -0,0 +1,255 @@
"""HypervisorDriver migration methods for Proxmox VE: preflight, migrate, HA maintenance.
A guest is addressed like every other VM method of this driver: by vmid or name,
on the node the driver is connected to (``_resolve_vm``).
- ``get_vm_migration_preflight``: a VM asks ``GET .../qemu/{vmid}/migrate``; a
container has no such endpoint, so the online members are its candidates.
A member the cluster calls offline is refused (``"node offline"``) for both,
also where the VM endpoint would list it as allowed.
- ``migrate_vm``: ``POST .../{qemu|lxc}/{vmid}/migrate`` and the task is followed
to its end; a failure raises with the task log's ``ERROR:`` lines. A guest HA
manages migrates through a ``hamigrate`` task that only hands the request to
the HA manager; the driver then follows ``/cluster/ha/status/current`` until
the guest is on the target, and on a failure reads the migration task's log.
- ``set_node_maintenance``: PVE has no REST endpoint for it; it runs
``ha-manager crm-command node-maintenance`` over the driver's SSH session.
The HA manager moves the node's HA guests away after the command returned.
"""
from __future__ import annotations
import logging
import shlex
from time import monotonic, sleep
from typing import TYPE_CHECKING, Any
from napalm_proxmox.migration import lxc_local, qemu_local, refusal, task_errors, upid_field
from napalm_proxmox.pve_values import to_flag
if TYPE_CHECKING:
# Type-only: newer than the napalm_device_types floor in pyproject.toml.
from napalm_device_types import MigrationPreflightDict
logger = logging.getLogger(__name__)
_JsonDict = dict[str, Any]
#: Enough to reach the ERROR lines at the end of a long storage migration's log.
_TASK_LOG_LINES = 50000
_HA_POLL_SECONDS = 5
#: CRM states while the HA manager is moving a guest.
_HA_MOVING = frozenset({"migrate", "relocate"})
class ProxmoxVMMigrateMixin:
"""Relies on the driver, ``_resolve_vm``/``_guest_api``, ``_members`` and
``_wait_for_task``/``_run_node_command``/``_node_ssh``."""
_api: Any
_node_name: str
hostname: str
timeout: int
_node_api: Any
_resolve_vm: Any
_guest_api: Any
_members: Any
_wait_for_task: Any
_node_ssh: Any
_run_node_command: Any
# -- cluster membership -------------------------------------------------------
def _cluster_members(self) -> list[Any]:
status = self._api.cluster.status.get() or []
clustered = any(s.get("type") == "cluster" for s in status)
return list(self._members(status if clustered else []))
def _other_member(self, node: str) -> None:
names = [m.name for m in self._cluster_members()]
if node == self._node_name or node not in names:
raise ValueError(
f"{node!r} is not another member of the cluster of {self._node_name!r} "
f"(members: {', '.join(names)})"
)
def _storage_sharing(self) -> dict[str, bool]:
rows = self._api.cluster.resources.get(type="storage") or []
sharing: dict[str, bool] = {}
for row in rows:
if row.get("type") == "storage" and row.get("storage"):
name = str(row["storage"])
sharing[name] = sharing.get(name, False) or to_flag(row.get("shared"))
return sharing
# -- preflight ----------------------------------------------------------------
def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict:
"""Where the guest could move to within its cluster, and what blocks it."""
vmid, vm_type = self._resolve_vm(name)
others = [m for m in self._cluster_members() if m.name != self._node_name]
offline = [m.name for m in others if not m.online]
if vm_type == "vm":
return self._qemu_preflight(vmid, offline)
return self._lxc_preflight(vmid, [m.name for m in others if m.online], offline)
def _qemu_preflight(self, vmid: int, offline: list[str]) -> MigrationPreflightDict:
answer = self._guest_api(vmid, "vm").migrate.get() or {}
refused = {
str(node): refusal(blockers)
for node, blockers in (answer.get("not_allowed_nodes") or {}).items()
}
for node in offline:
refused[node] = "; ".join(filter(None, ["node offline", refused.get(node)]))
disks, resources = qemu_local(answer)
return {
"running": to_flag(answer.get("running")),
"allowed_nodes": [
str(n) for n in answer.get("allowed_nodes") or [] if n not in offline
],
"refused_nodes": refused,
"local_disks": disks,
"local_resources": resources,
}
def _lxc_running(self, vmid: int) -> bool:
status = self._guest_api(vmid, "container").status.current.get() or {}
return bool(status.get("status") == "running")
def _lxc_preflight(
self, vmid: int, online: list[str], offline: list[str]
) -> MigrationPreflightDict:
config = self._guest_api(vmid, "container").config.get() or {}
disks, resources = lxc_local(config, self._storage_sharing())
return {
"running": self._lxc_running(vmid),
"allowed_nodes": online,
"refused_nodes": {n: "node offline" for n in offline},
"local_disks": disks,
"local_resources": resources,
}
# -- migration ----------------------------------------------------------------
def migrate_vm(
self,
name: str,
target_node: str,
online: bool = True,
target_storage: str | None = None,
timeout: int = 3600,
) -> None:
"""Move the guest to *target_node*; block until Proxmox VE finished."""
vmid, vm_type = self._resolve_vm(name)
self._other_member(target_node)
deadline = monotonic() + timeout
params = self._qemu_params(vmid, online) if vm_type == "vm" else self._lxc_params(vmid)
if target_storage:
params["targetstorage" if vm_type == "vm" else "target-storage"] = target_storage
try:
upid = str(self._guest_api(vmid, vm_type).migrate.post(target=target_node, **params))
except Exception as exc:
raise RuntimeError(
f"Migration of {vm_type} {vmid} to {target_node!r} refused: {exc}"
) from exc
self._follow_task(upid, vmid, timeout)
if upid_field(upid, 5) == "hamigrate":
self._follow_ha(vmid, vm_type, target_node, deadline, upid)
def _qemu_params(self, vmid: int, online: bool) -> dict[str, Any]:
answer = self._guest_api(vmid, "vm").migrate.get() or {}
params: dict[str, Any] = {}
if online and to_flag(answer.get("running")):
params["online"] = 1
if qemu_local(answer)[0]:
params["with-local-disks"] = 1
return params
def _lxc_params(self, vmid: int) -> dict[str, Any]:
# A container cannot move while running; restart mode stops and restarts it.
return {"restart": 1} if self._lxc_running(vmid) else {}
def _task_errors(self, upid: str) -> list[str]:
try:
log = self._node_api().tasks(upid).log.get(limit=_TASK_LOG_LINES) or []
except Exception as exc:
logger.debug("Cannot read the log of task %s: %s", upid, exc)
return []
return task_errors(log)
def _follow_task(self, upid: str, vmid: int, timeout: int) -> None:
try:
self._wait_for_task(upid, timeout=timeout)
except RuntimeError as exc:
errors = self._task_errors(upid)
if not errors:
raise
raise RuntimeError(f"Migration of guest {vmid} failed: {'; '.join(errors)}") from exc
# -- a guest HA manages -------------------------------------------------------
def _ha_state(self, sid: str) -> tuple[str | None, str | None]:
rows = self._api.cluster.ha.status.current.get() or []
service: _JsonDict = next((r for r in rows if r.get("sid") == sid), {})
return service.get("node"), service.get("crm_state") or service.get("state")
def _follow_ha(
self, vmid: int, vm_type: str, target: str, deadline: float, ha_upid: str
) -> None:
sid = f"{'vm' if vm_type == 'vm' else 'ct'}:{vmid}"
moving_seen = False
while True:
node, state = self._ha_state(sid)
if node == target and state not in _HA_MOVING:
return
if state == "error":
self._ha_failed(vmid, vm_type, ha_upid, f"HA manager put {sid} into error state")
if state in _HA_MOVING:
moving_seen = True
elif moving_seen:
self._ha_failed(
vmid, vm_type, ha_upid, f"HA manager left {sid} on {node} ({state})"
)
if monotonic() > deadline:
raise RuntimeError(
f"Migration of {sid} to {target!r} timed out on {node} ({state})"
)
sleep(_HA_POLL_SECONDS)
def _ha_failed(self, vmid: int, vm_type: str, ha_upid: str, what: str) -> None:
"""Raise *what* with the ERROR lines of the migration the HA manager ran."""
errors: list[str] = []
kind = "qmigrate" if vm_type == "vm" else "vzmigrate"
try:
tasks = self._node_api().tasks.get(vmid=vmid, typefilter=kind, limit=1) or []
since = int(upid_field(ha_upid, 4) or "0", 16)
if tasks and int(tasks[0].get("starttime") or 0) >= since:
errors = self._task_errors(str(tasks[0].get("upid")))
except Exception as exc:
logger.debug("Cannot find the migration task of guest %s: %s", vmid, exc)
raise RuntimeError(f"{what}: {'; '.join(errors)}" if errors else what)
# -- HA maintenance -----------------------------------------------------------
def set_node_maintenance(self, node: str, enabled: bool) -> None:
"""Switch HA maintenance mode of a cluster node, over SSH (no REST endpoint)."""
names = [m.name for m in self._cluster_members()]
if node not in names:
raise ValueError(
f"{node!r} is not a member of the cluster (members: {', '.join(names)})"
)
action = "enable" if enabled else "disable"
try:
self._node_ssh()
except Exception as exc:
raise RuntimeError(
f"Cannot {action} HA maintenance of node {node!r}: SSH to {self.hostname} "
f"is not available ({exc})"
) from exc
command = f"ha-manager crm-command node-maintenance {action} {shlex.quote(node)}"
try:
self._run_node_command(command, timeout=self.timeout)
except Exception as exc:
raise RuntimeError(
f"ha-manager refused to {action} maintenance of {node!r}: {exc}"
) from exc
+555
View File
@@ -0,0 +1,555 @@
"""get_clusters(): quorum votes, the HA manager's guests, backup jobs and backups.
Each of these is read on top of the cluster report and is best-effort: a read
that fails leaves its key None and the rest of the report stands.
Payloads: ``/cluster/config/nodes`` is corosync's nodelist; ``/cluster/ha/*``
follow pve-ha-manager's API (``src/PVE/API2/HA/{Resources,Status,Rules}.pm``,
master = PVE 9 and stable-4 = PVE 8): a service entry of ``status/current``
carries ``crm_state`` and ``request_state``, a rule carries ``rule``, ``type``
and ``resources`` as a sorted, comma-separated list of sids, and PVE 8 has no
``/cluster/ha/rules``. ``/cluster/backup`` follows pve-manager's
``PVE/API2/Backup.pm``; storage content rows follow
``/nodes/{node}/storage/{storage}/content``.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
from tests.test_clusters import (
CLUSTER_RESOURCES,
GIB,
_api,
_by_name,
_driver,
_storage_row,
)
def _report(api: MagicMock) -> dict[str, Any]:
[cluster] = _driver(api).get_clusters()
return dict(cluster)
class TestVotes:
def test_each_node_has_its_quorum_votes(self):
nodes = _by_name(_report(_api())["nodes"])
assert [nodes[n]["votes"] for n in ("pve1", "pve2", "pve3")] == [1, 1, 2]
def test_unreadable_nodelist_leaves_votes_unknown(self):
api = _api()
api.cluster.config.nodes.get.side_effect = RuntimeError("403 Forbidden")
nodes = _report(api)["nodes"]
assert [n["votes"] for n in nodes] == [None, None, None]
# The rest of the node still stands.
assert nodes[0]["cpu_threads"] == 32
def test_standalone_node_has_no_votes(self):
# A node without corosync.conf answers with an empty nodelist.
api = _api()
api.cluster.config.nodes.get.return_value = []
assert [n["votes"] for n in _report(api)["nodes"]] == [None, None, None]
def test_node_without_quorum_votes_counts_the_default(self):
"""corosync counts a node without quorum_votes as one vote."""
api = _api()
api.cluster.config.nodes.get.return_value = [{"node": "pve1", "name": "pve1"}]
nodes = _by_name(_report(api)["nodes"])
assert nodes["pve1"]["votes"] == 1
assert nodes["pve2"]["votes"] is None
# --------------------------------------------------------------------------- HA
HA_RESOURCES_PVE8 = [
{
"sid": "vm:100",
"type": "vm",
"state": "started",
"group": "prefer-pve1",
"max_restart": 1,
"max_relocate": 1,
"digest": "3f1c",
},
# No state in the config: HA's default, "started".
{"sid": "ct:101", "type": "ct", "digest": "3f1c"},
{"sid": "vm:102", "type": "vm", "state": "disabled", "digest": "3f1c"},
]
HA_STATUS = [
{"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"},
{
"id": "master",
"type": "master",
"node": "pve1",
"status": "pve1 (active, Sun Nov 1 10:00:00 2026)",
"timestamp": 1793527200,
},
{
"id": "lrm:pve1",
"type": "lrm",
"node": "pve1",
"status": "pve1 (active, Sun Nov 1 10:00:00 2026)",
"timestamp": 1793527200,
},
{
"id": "service:vm:100",
"type": "service",
"sid": "vm:100",
"node": "pve2",
"crm_state": "migrate",
"state": "migrate",
"status": "vm:100 (pve2, migrate)",
"request_state": "started",
"group": "prefer-pve1",
"max_restart": 1,
"max_relocate": 1,
},
{
"id": "service:ct:101",
"type": "service",
"sid": "ct:101",
"node": "pve1",
"crm_state": "started",
"state": "started",
"status": "ct:101 (pve1, started)",
"request_state": "started",
},
# Not yet processed by the CRM: no crm_state, a verbose state only.
{
"id": "service:vm:102",
"type": "service",
"sid": "vm:102",
"state": "queued",
"status": "vm:102 (---, queued)",
"request_state": "disabled",
},
]
# PVE 9: groups were migrated into node-affinity rules, so resources lose "group".
HA_RESOURCES_PVE9 = [{k: v for k, v in r.items() if k != "group"} for r in HA_RESOURCES_PVE8]
HA_RULES_PVE9 = [
{
"rule": "ha-group-prefer-pve1",
"type": "node-affinity",
"resources": "vm:100",
"nodes": "pve1:2,pve2:1",
"strict": 0,
"digest": "9ac2",
"order": 1,
},
{
"rule": "keep-apart",
"type": "resource-affinity",
"affinity": "negative",
"resources": "ct:101,vm:100",
"digest": "9ac2",
"order": 2,
},
]
def _ha_api(
resources: Any = HA_RESOURCES_PVE8,
status: Any = HA_STATUS,
rules: Any = None,
) -> MagicMock:
api = _api()
ha = api.cluster.ha
for endpoint, value in ((ha.resources, resources), (ha.status.current, status)):
if isinstance(value, Exception):
endpoint.get.side_effect = value
else:
endpoint.get.return_value = value
if rules is None:
# PVE 8 has no rules endpoint.
ha.rules.get.side_effect = RuntimeError(
"501 Not Implemented: Method 'GET /cluster/ha/rules' not implemented"
)
else:
ha.rules.get.return_value = rules
return api
def _ha(api: MagicMock) -> dict[int, dict[str, Any]] | None:
resources = _report(api)["ha_resources"]
return None if resources is None else {r["vmid"]: r for r in resources}
class TestHAResources:
def test_pve8_resource_with_group(self):
ha = _ha(_ha_api())
assert ha is not None
assert ha[100] == {
"vmid": 100,
"kind": "qemu",
"requested": "started",
"status": "migrate",
"node": "pve2",
"group": "prefer-pve1",
"rules": [],
}
def test_container_and_default_requested_state(self):
ha = _ha(_ha_api())
assert ha is not None
assert ha[101]["kind"] == "lxc"
assert ha[101]["requested"] == "started"
assert ha[101]["status"] == "started"
assert ha[101]["node"] == "pve1"
assert ha[101]["group"] is None
def test_not_yet_processed_reports_the_verbose_state(self):
ha = _ha(_ha_api())
assert ha is not None
assert ha[102]["requested"] == "disabled"
assert ha[102]["status"] == "queued"
assert ha[102]["node"] is None
def test_pve9_rules_name_their_resources(self):
ha = _ha(_ha_api(resources=HA_RESOURCES_PVE9, rules=HA_RULES_PVE9))
assert ha is not None
assert ha[100]["rules"] == ["ha-group-prefer-pve1", "keep-apart"]
assert ha[101]["rules"] == ["keep-apart"]
assert ha[102]["rules"] == []
assert ha[100]["group"] is None
def test_no_ha_configured_is_an_empty_list(self):
assert _report(_ha_api(resources=[], status=[]))["ha_resources"] == []
def test_unreadable_resources_leave_the_list_unknown(self):
api = _ha_api(resources=RuntimeError("403 Forbidden: Permission check failed"))
report = _report(api)
assert report["ha_resources"] is None
assert report["name"] == "lab"
def test_unreadable_status_leaves_status_and_node_unknown(self):
ha = _ha(_ha_api(status=RuntimeError("500 cfs lock")))
assert ha is not None
assert (ha[100]["status"], ha[100]["node"]) == (None, None)
assert ha[100]["requested"] == "started"
# ---------------------------------------------------------------------- backups
BACKUP_JOBS = [
{
"id": "backup-5a1b2c3d-e4f5",
"type": "vzdump",
"enabled": 1,
"schedule": "sat 02:00",
"storage": "pbs",
"mode": "snapshot",
"all": 1,
"exclude": "105,106",
"compress": "zstd",
"mailnotification": "always",
"next-run": 1793584800,
"notes-template": "{{guestname}}",
},
{
"id": "backup-7e8f9a0b-1c2d",
"type": "vzdump",
"enabled": 0,
"schedule": "mon..fri 21:00",
"storage": "local",
"mode": "stop",
"vmid": "100,101",
"node": "pve1",
},
# No "enabled": a job is enabled unless it says otherwise.
{
"id": "backup-0b1c2d3e-4f5a",
"type": "vzdump",
"schedule": "daily",
"storage": "pbs",
"mode": "suspend",
"pool": "prod",
},
]
# PVE >= 8.1: GET /pools?poolid=prod
POOL_INDEX = [
{
"poolid": "prod",
"comment": "",
"members": [
{"id": "qemu/100", "type": "qemu", "vmid": 100, "node": "pve1", "name": "web"},
{"id": "lxc/101", "type": "lxc", "vmid": 101, "node": "pve1", "name": "dns"},
{"id": "storage/pve1/local", "type": "storage", "storage": "local", "node": "pve1"},
],
}
]
def _backup_api(jobs: Any = BACKUP_JOBS) -> MagicMock:
api = _api()
if isinstance(jobs, Exception):
api.cluster.backup.get.side_effect = jobs
else:
api.cluster.backup.get.return_value = jobs
api.pools.get.return_value = POOL_INDEX
return api
def _jobs(api: MagicMock) -> dict[str, dict[str, Any]]:
jobs = _report(api)["backup_jobs"]
assert jobs is not None
return {j["id"]: j for j in jobs}
class TestBackupJobs:
def test_all_guests_job(self):
assert _jobs(_backup_api())["backup-5a1b2c3d-e4f5"] == {
"id": "backup-5a1b2c3d-e4f5",
"enabled": True,
"schedule": "sat 02:00",
"storage": "pbs",
"mode": "snapshot",
"all": True,
"vmids": [],
"exclude": [105, 106],
"pool": None,
"node": None,
}
def test_disabled_job_for_named_guests_on_one_node(self):
job = _jobs(_backup_api())["backup-7e8f9a0b-1c2d"]
assert job["enabled"] is False
assert job["all"] is False
assert job["vmids"] == [100, 101]
assert job["exclude"] == []
assert job["node"] == "pve1"
def test_pool_job_lists_the_pools_guests(self):
api = _backup_api()
job = _jobs(api)["backup-0b1c2d3e-4f5a"]
assert job["enabled"] is True
assert job["pool"] == "prod"
assert job["vmids"] == [100, 101]
api.pools.get.assert_called_with(poolid="prod")
def test_pool_read_falls_back_to_the_old_endpoint(self):
"""Before PVE 8.1, /pools takes no poolid; /pools/{poolid} answers instead."""
api = _backup_api()
api.pools.get.side_effect = RuntimeError("400 Bad Request: Parameter verification failed")
pool = MagicMock()
pool.get.return_value = {"comment": "", "members": POOL_INDEX[0]["members"]}
api.pools.side_effect = lambda name: pool if name == "prod" else MagicMock()
assert _jobs(api)["backup-0b1c2d3e-4f5a"]["vmids"] == [100, 101]
def test_old_style_start_time_and_days(self):
"""Jobs from vzdump.cron carry starttime/dow; PVE turns them into a schedule."""
jobs = [
{"id": "a", "starttime": "02:30", "dow": "sat,sun", "storage": "local", "vmid": "102"},
{
"id": "b",
"starttime": "03:00",
"dow": "mon,tue,wed,thu,fri,sat,sun",
"storage": "local",
"all": 1,
},
{"id": "c", "starttime": "04:00", "storage": "local", "all": 1},
]
found = _jobs(_backup_api(jobs))
assert [found[i]["schedule"] for i in "abc"] == ["sat,sun 02:30", "03:00", "04:00"]
def test_job_with_a_dump_directory_has_no_storage(self):
jobs = [{"id": "d", "schedule": "daily", "dumpdir": "/mnt/backup", "all": 1}]
job = _jobs(_backup_api(jobs))["d"]
assert job["storage"] is None
assert job["mode"] is None
def test_no_jobs_is_an_empty_list(self):
assert _report(_backup_api([]))["backup_jobs"] == []
def test_unreadable_jobs_leave_the_list_unknown(self):
report = _report(_backup_api(RuntimeError("403 Forbidden")))
assert report["backup_jobs"] is None
assert report["nodes"]
def test_unreadable_pool_leaves_the_list_unknown(self):
"""A pool job whose guests cannot be read would look like a job for none."""
api = _backup_api()
api.pools.get.side_effect = RuntimeError("403 Forbidden")
api.pools.return_value.get.side_effect = RuntimeError("403 Forbidden")
api.pools.side_effect = None
assert _report(api)["backup_jobs"] is None
BACKUP_RESOURCES = [
*CLUSTER_RESOURCES,
_storage_row("pve1", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB),
_storage_row("pve2", "pbs", "pbs", 1, "backup", 8000 * GIB, 2000 * GIB),
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB),
_storage_row("pve2", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 30 * GIB),
_storage_row("pve3", "local", "dir", 0, "iso,vztmpl,backup", 0, 0, status="unknown"),
]
PBS_CONTENT = [
{
"volid": "pbs:backup/vm/100/2026-10-31T01:00:00Z",
"content": "backup",
"format": "pbs-vm",
"ctime": 1793408400,
"size": 32 * GIB,
"vmid": 100,
"subtype": "qemu",
"notes": "web",
"protected": 0,
"verification": {"state": "ok", "upid": "UPID:pbs:0000:0000:0000:verify:x:root@pam:"},
},
{
"volid": "pbs:backup/vm/100/2026-10-24T01:00:00Z",
"content": "backup",
"format": "pbs-vm",
"ctime": 1792803600,
"size": 31 * GIB,
"vmid": 100,
"subtype": "qemu",
},
{
"volid": "pbs:backup/ct/101/2026-10-31T01:05:00Z",
"content": "backup",
"format": "pbs-ct",
"ctime": 1793408700,
"size": 2 * GIB,
"vmid": 101,
"subtype": "lxc",
},
]
LOCAL_CONTENT = {
"pve1": [
{
"volid": "local:backup/vzdump-qemu-100-2026_10_30-21_00_02.vma.zst",
"content": "backup",
"format": "vma.zst",
"ctime": 1793394002,
"size": 1234567,
"vmid": 100,
"subtype": "qemu",
},
# A file that does not follow vzdump's naming has no vmid.
{"volid": "local:backup/manual.tar", "content": "backup", "format": "tar", "size": 10},
],
"pve2": [
{
"volid": "local:backup/vzdump-qemu-100-2026_10_29-21_00_01.vma.zst",
"content": "backup",
"format": "vma.zst",
"ctime": 1793307601,
"size": 1200000,
"vmid": 100,
"subtype": "qemu",
},
],
}
def _storage_api(
content: dict[tuple[str, str], Any] | None = None,
) -> tuple[MagicMock, dict[tuple[str, str], MagicMock]]:
"""The cluster with backup storages; *content* per (node, storage)."""
rows = content or {
("pve1", "pbs"): PBS_CONTENT,
("pve2", "pbs"): PBS_CONTENT,
("pve1", "local"): LOCAL_CONTENT["pve1"],
("pve2", "local"): LOCAL_CONTENT["pve2"],
}
api = _api(resources=BACKUP_RESOURCES)
calls: dict[tuple[str, str], MagicMock] = {}
for node in ("pve1", "pve2"):
def storage(name: str, node: str = node) -> MagicMock:
res = calls.setdefault((node, name), MagicMock())
value = rows.get((node, name), [])
if isinstance(value, Exception):
res.content.get.side_effect = value
else:
res.content.get.return_value = value
return res
api.nodes(node).storage.side_effect = storage
return api, calls
def _backups(api: MagicMock) -> dict[tuple[int, str], dict[str, Any]] | None:
backups = _report(api)["backups"]
return None if backups is None else {(b["vmid"], b["storage"]): b for b in backups}
class TestBackups:
def test_newest_backup_per_guest_and_storage(self):
api, _ = _storage_api()
backups = _backups(api)
assert backups is not None
assert backups[(100, "pbs")] == {
"vmid": 100,
"storage": "pbs",
"time": 1793408400,
"size": 32 * GIB,
"count": 2,
}
assert backups[(101, "pbs")]["count"] == 1
def test_shared_storage_read_once_on_the_own_node(self):
api, calls = _storage_api()
_report(api)
calls[("pve1", "pbs")].content.get.assert_called_once_with(content="backup")
assert ("pve2", "pbs") not in calls
def test_local_storage_read_on_each_node_that_has_it(self):
api, calls = _storage_api()
backups = _backups(api)
assert backups is not None
# The same guest backed up to "local" on two nodes: counted together.
assert backups[(100, "local")] == {
"vmid": 100,
"storage": "local",
"time": 1793394002,
"size": 1234567,
"count": 2,
}
assert ("pve1", "local") in calls and ("pve2", "local") in calls
def test_storage_without_backup_content_is_not_read(self):
api, calls = _storage_api()
_report(api)
assert not [key for key in calls if key[1] in ("ceph-vm", "local-lvm", "nfs-iso")]
def test_entry_without_vmid_is_skipped(self):
api, _ = _storage_api()
backups = _backups(api)
assert backups is not None
assert {vmid for vmid, _ in backups} == {100, 101}
def test_shared_storage_tried_on_the_next_node(self):
api, _ = _storage_api(
{
("pve1", "pbs"): RuntimeError("pbs: datastore not reachable from pve1"),
("pve2", "pbs"): PBS_CONTENT,
}
)
backups = _backups(api)
assert backups is not None
assert backups[(100, "pbs")]["count"] == 2
def test_unreadable_storage_leaves_backups_unknown(self):
"""Half a list would say a guest has no backup when it may have one."""
api, _ = _storage_api(
{
("pve1", "pbs"): PBS_CONTENT,
("pve1", "local"): LOCAL_CONTENT["pve1"],
("pve2", "local"): RuntimeError("596 Connection timed out"),
}
)
report = _report(api)
assert report["backups"] is None
assert report["storages"]
def test_no_backup_storage_is_an_empty_list(self):
assert _report(_api())["backups"] == []
+14
View File
@@ -167,6 +167,13 @@ SDN_VNETS = [
{"vnet": "overlay", "zone": "fabric", "tag": 100000, "type": "vnet", "digest": "x"},
]
# /cluster/config/nodes: corosync's nodelist; corosync.conf keeps every value a string.
CONFIG_NODES = [
{"node": "pve1", "name": "pve1", "nodeid": "1", "quorum_votes": "1", "ring0_addr": "10.0.0.1"},
{"node": "pve2", "name": "pve2", "nodeid": "2", "quorum_votes": "1", "ring0_addr": "10.0.0.2"},
{"node": "pve3", "name": "pve3", "nodeid": "3", "quorum_votes": "2", "ring0_addr": "10.0.0.3"},
]
HA_MANAGER_STATUS = {
"manager_status": {
"master_node": "pve1",
@@ -213,6 +220,12 @@ def _api(
manager.side_effect = ha
else:
manager.return_value = HA_MANAGER_STATUS if ha is None else ha
# Read for get_clusters' votes, HA resources and backups; see test_cluster_ha_backups.
api.cluster.config.nodes.get.return_value = CONFIG_NODES
api.cluster.ha.resources.get.return_value = []
api.cluster.ha.status.current.get.return_value = []
api.cluster.ha.rules.get.return_value = []
api.cluster.backup.get.return_value = []
return api
@@ -268,6 +281,7 @@ class TestNodes:
"cpu_usage": 0.25,
"memory_total": 128 * GIB,
"memory_used": 64 * GIB,
"votes": 1,
}
def test_other_member_is_not_local(self):
+382
View File
@@ -0,0 +1,382 @@
"""Moving a guest to another node: preflight, migration, HA maintenance mode.
Payloads: ``GET /nodes/{node}/qemu/{vmid}/migrate`` follows qemu-server's
``migrate_vm_precondition`` (PVE 9: ``not_allowed_nodes`` may name
``unavailable_storages``, ``unavailable-resources`` and
``blocking-ha-resources``); the task log follows ``/nodes/{node}/tasks/{upid}/log``
(``n``/``t`` rows). A guest that HA manages migrates through a ``hamigrate``
task that only hands the request to the HA manager.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from tests.test_clusters import CLUSTER_RESOURCES, CLUSTER_STATUS, GIB, _api, _driver, _storage_row
QEMU_LIST = [
{"vmid": 100, "name": "web", "status": "running", "maxmem": 4 * GIB},
{"vmid": 102, "name": "idle", "status": "stopped", "maxmem": 2 * GIB},
]
LXC_LIST = [{"vmid": "101", "name": "dns", "status": "running", "type": "lxc"}]
PREFLIGHT_RUNNING = {
"running": 1,
"allowed_nodes": ["pve2", "pve3"],
"not_allowed_nodes": {},
"local_disks": [
{
"volid": "local-lvm:vm-100-disk-0",
"size": 32 * GIB,
"drivename": "scsi0",
"is_attached": 1,
"referenced_in_config": 1,
},
{"volid": "local:iso/debian-13.iso", "cdrom": 1, "size": 700 * 1024**2},
# Regenerated on the target: neither a disk to copy nor a blocker.
{"volid": "local-lvm:vm-100-cloudinit", "cdrom": 1, "size": 4 * 1024**2},
],
"local_resources": [],
"mapped-resources": [],
"mapped-resource-info": {},
"has-dbus-vmstate": 1,
}
PREFLIGHT_BLOCKED = {
"running": 0,
"allowed_nodes": [],
"not_allowed_nodes": {
"pve2": {"unavailable_storages": ["local-zfs"]},
"pve3": {
"blocking-ha-resources": [{"sid": "vm:101", "cause": "resource-affinity"}],
"unavailable-resources": ["gpu-a"],
},
},
"local_disks": [],
"local_resources": ["hostpci0", "usb0"],
"mapped-resources": ["gpu-a"],
"mapped-resource-info": {"gpu-a": {"live-migration": 0}},
"has-dbus-vmstate": 1,
}
LXC_CONFIG = {
"hostname": "dns",
"rootfs": "local-lvm:vm-101-disk-0,size=8G",
"mp0": "ceph-vm:vm-101-disk-1,mp=/data,size=32G",
"mp1": "/srv/share,mp=/share",
"mp2": "/mnt/nfs,mp=/nfs,shared=1",
"unused0": "local-lvm:vm-101-disk-2",
"net0": "name=eth0,bridge=vmbr0,ip=dhcp",
"digest": "abc",
}
UPID_QEMU = "UPID:pve1:000ABCDE:0123ABCD:672AB000:qmigrate:100:root@pam:"
UPID_LXC = "UPID:pve1:000ABCDF:0123ABCE:672AB001:vzmigrate:101:root@pam:"
UPID_HA = "UPID:pve1:000ABCE0:0123ABCF:672AB002:hamigrate:100:root@pam:"
FAILED_LOG = [
{"n": 1, "t": "2026-11-01 10:00:00 starting migration of VM 100 to node 'pve2' (10.0.0.2)"},
{"n": 2, "t": "2026-11-01 10:00:01 starting VM 100 on remote node 'pve2'"},
{
"n": 3,
"t": "2026-11-01 10:00:02 ERROR: online migrate failure - remote command failed "
"with exit code 255",
},
{"n": 4, "t": "2026-11-01 10:00:02 aborting phase 2 - cleanup resources"},
{
"n": 5,
"t": "2026-11-01 10:00:03 ERROR: migration finished with problems (duration 00:00:03)",
},
{"n": 6, "t": "TASK ERROR: migration problems"},
]
def _migration_api(
preflight: dict[str, Any] | None = None,
task: dict[str, Any] | None = None,
log: list[dict[str, Any]] | None = None,
status: list[dict[str, Any]] | None = None,
) -> tuple[MagicMock, MagicMock]:
resources = [
*CLUSTER_RESOURCES,
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB),
]
api = _api(cluster_status=status, resources=resources)
node = api.nodes("pve1")
node.qemu.get.return_value = QEMU_LIST
node.lxc.get.return_value = LXC_LIST
node.qemu.return_value.migrate.get.return_value = preflight or PREFLIGHT_RUNNING
node.qemu.return_value.migrate.post.return_value = UPID_QEMU
node.lxc.return_value.migrate.post.return_value = UPID_LXC
node.lxc.return_value.status.current.get.return_value = {"status": "running", "vmid": 101}
node.lxc.return_value.config.get.return_value = LXC_CONFIG
node.tasks.return_value.status.get.return_value = task or {
"status": "stopped",
"exitstatus": "OK",
}
node.tasks.return_value.log.get.return_value = log or []
return api, node
class TestPreflightQemu:
def test_allowed_nodes_and_local_disks(self):
api, node = _migration_api()
result = _driver(api).get_vm_migration_preflight("web")
assert result == {
"running": True,
# pve3 is offline: the platform would allow it, it cannot take the VM now.
"allowed_nodes": ["pve2"],
"refused_nodes": {"pve3": "node offline"},
"local_disks": ["local-lvm:vm-100-disk-0"],
# A local ISO in a CD drive stops the migration.
"local_resources": ["cdrom=local:iso/debian-13.iso"],
}
node.qemu.assert_any_call(100)
def test_refusal_reasons_in_plain_words(self):
all_online = [dict(s, online=1) if s["type"] == "node" else s for s in CLUSTER_STATUS]
api, _ = _migration_api(preflight=PREFLIGHT_BLOCKED, status=all_online)
result = _driver(api).get_vm_migration_preflight("100")
assert result["running"] is False
assert result["allowed_nodes"] == []
assert result["refused_nodes"] == {
"pve2": "storage local-zfs is not available there",
"pve3": "HA resource vm:101 blocks it (resource-affinity rule); "
"mapped resource gpu-a is not available there",
}
assert result["local_resources"] == ["hostpci0", "usb0"]
def test_unknown_reason_is_passed_on(self):
preflight = {
"running": 0,
"allowed_nodes": [],
"not_allowed_nodes": {"pve2": {"some-future-check": ["x", "y"]}},
"local_disks": [],
"local_resources": [],
}
api, _ = _migration_api(preflight=preflight)
refused = _driver(api).get_vm_migration_preflight("web")["refused_nodes"]
assert refused["pve2"] == "some-future-check: x, y"
def test_unknown_guest(self):
api, _ = _migration_api()
with pytest.raises(ValueError, match="nope"):
_driver(api).get_vm_migration_preflight("nope")
class TestPreflightLxc:
def test_members_and_local_volumes(self):
api, node = _migration_api()
result = _driver(api).get_vm_migration_preflight("dns")
assert result == {
"running": True,
"allowed_nodes": ["pve2"],
"refused_nodes": {"pve3": "node offline"},
# ceph-vm is shared; local-lvm is not.
"local_disks": ["local-lvm:vm-101-disk-0", "local-lvm:vm-101-disk-2"],
# A bind mount that is not marked shared stops the migration.
"local_resources": ["mp1"],
}
node.qemu.return_value.migrate.get.assert_not_called()
def test_stopped_container(self):
api, node = _migration_api()
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
assert _driver(api).get_vm_migration_preflight("101")["running"] is False
class TestMigrateQemu:
def test_running_vm_moves_online_with_its_local_disks(self):
api, node = _migration_api()
_driver(api).migrate_vm("web", "pve2")
node.qemu.return_value.migrate.post.assert_called_once_with(
target="pve2", online=1, **{"with-local-disks": 1}
)
node.tasks.assert_called_with(UPID_QEMU)
def test_target_storage_and_offline(self):
preflight = dict(PREFLIGHT_RUNNING, local_disks=[])
api, node = _migration_api(preflight=preflight)
_driver(api).migrate_vm("web", "pve2", online=False, target_storage="ceph-vm")
node.qemu.return_value.migrate.post.assert_called_once_with(
target="pve2", targetstorage="ceph-vm"
)
def test_stopped_vm_is_not_moved_online(self):
preflight = dict(PREFLIGHT_RUNNING, running=0, local_disks=[])
api, node = _migration_api(preflight=preflight)
_driver(api).migrate_vm("idle", "pve2")
node.qemu.return_value.migrate.post.assert_called_once_with(target="pve2")
def test_failure_raises_with_the_task_logs_errors(self):
api, node = _migration_api(
task={"status": "stopped", "exitstatus": "migration problems"}, log=FAILED_LOG
)
with pytest.raises(RuntimeError) as exc:
_driver(api).migrate_vm("web", "pve2")
message = str(exc.value)
assert "online migrate failure - remote command failed with exit code 255" in message
assert "migration finished with problems (duration 00:00:03)" in message
assert "starting VM 100" not in message
node.tasks.return_value.log.get.assert_called_once()
def test_failure_without_error_lines_keeps_the_task_status(self):
api, _ = _migration_api(
task={"status": "stopped", "exitstatus": "migration aborted"},
log=[{"n": 1, "t": "starting migration"}],
)
with pytest.raises(RuntimeError, match="migration aborted"):
_driver(api).migrate_vm("web", "pve2")
def test_refused_request_raises(self):
api, node = _migration_api()
node.qemu.return_value.migrate.post.side_effect = RuntimeError(
"500 Internal Server Error: VM is locked (backup)"
)
with pytest.raises(RuntimeError, match="VM is locked"):
_driver(api).migrate_vm("web", "pve2")
class TestMigrateLxc:
def test_running_container_moves_in_restart_mode(self):
api, node = _migration_api()
_driver(api).migrate_vm("dns", "pve2", target_storage="local-lvm")
node.lxc.return_value.migrate.post.assert_called_once_with(
target="pve2", restart=1, **{"target-storage": "local-lvm"}
)
node.tasks.assert_called_with(UPID_LXC)
def test_stopped_container(self):
api, node = _migration_api()
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
_driver(api).migrate_vm("dns", "pve2")
node.lxc.return_value.migrate.post.assert_called_once_with(target="pve2")
class TestTargetRefusedBeforeTheApi:
@pytest.mark.parametrize("target", ["pve1", "pve9", "pve2; reboot"])
def test_not_another_member(self, target):
api, node = _migration_api()
with pytest.raises(ValueError, match="not another member"):
_driver(api).migrate_vm("web", target)
node.qemu.return_value.migrate.post.assert_not_called()
def _ha_status(node: str, crm_state: str) -> list[dict[str, Any]]:
return [
{"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"},
{
"id": "service:vm:100",
"type": "service",
"sid": "vm:100",
"node": node,
"crm_state": crm_state,
"state": crm_state,
"status": f"vm:100 ({node}, {crm_state})",
"request_state": "started",
},
]
class TestMigrateHaManaged:
"""PVE hands an HA guest's migration to the HA manager: the hamigrate task only
queues the request, so the driver follows the HA manager until the guest is there."""
def _api(self, states: list[list[dict[str, Any]]]) -> tuple[MagicMock, MagicMock]:
api, node = _migration_api()
node.qemu.return_value.migrate.post.return_value = UPID_HA
api.cluster.ha.status.current.get.side_effect = states
return api, node
def test_waits_until_the_guest_runs_on_the_target(self):
api, _ = self._api(
[
_ha_status("pve1", "started"),
_ha_status("pve1", "migrate"),
_ha_status("pve2", "started"),
]
)
with patch("napalm_proxmox.vm_migrate_mixin.sleep") as sleep:
_driver(api).migrate_vm("web", "pve2")
assert api.cluster.ha.status.current.get.call_count == 3
assert sleep.call_count == 2
def test_back_on_the_source_node_is_a_failure_with_the_migration_log(self):
api, node = self._api([_ha_status("pve1", "migrate"), _ha_status("pve1", "started")])
node.tasks.get.return_value = [
{
"upid": UPID_QEMU,
"type": "qmigrate",
"id": "100",
"starttime": 0x672AB002 + 12,
"status": "migration problems",
}
]
node.tasks.return_value.log.get.return_value = FAILED_LOG
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
with pytest.raises(RuntimeError, match="online migrate failure"):
_driver(api).migrate_vm("web", "pve2")
node.tasks.get.assert_called_once_with(vmid=100, typefilter="qmigrate", limit=1)
def test_error_state_raises(self):
api, node = self._api([_ha_status("pve1", "error")])
node.tasks.get.return_value = []
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
with pytest.raises(RuntimeError, match="error"):
_driver(api).migrate_vm("web", "pve2")
def test_times_out(self):
api, _ = self._api([_ha_status("pve1", "migrate")] * 50)
clock = iter(range(0, 10000, 30))
with (
patch("napalm_proxmox.vm_migrate_mixin.sleep"),
patch("napalm_proxmox.vm_migrate_mixin.monotonic", lambda: next(clock)),
):
with pytest.raises(RuntimeError, match="timed out"):
_driver(api).migrate_vm("web", "pve2", timeout=120)
class TestNodeMaintenance:
def _driver(self) -> tuple[Any, MagicMock]:
api, _ = _migration_api()
drv = _driver(api)
drv._node_ssh = MagicMock() # type: ignore[method-assign]
drv._run_node_command = MagicMock(return_value="") # type: ignore[method-assign]
return drv, drv._run_node_command
def test_enable(self):
drv, run = self._driver()
drv.set_node_maintenance("pve2", True)
run.assert_called_once_with(
"ha-manager crm-command node-maintenance enable pve2", timeout=drv.timeout
)
def test_disable(self):
drv, run = self._driver()
drv.set_node_maintenance("pve3", False)
run.assert_called_once_with(
"ha-manager crm-command node-maintenance disable pve3", timeout=drv.timeout
)
@pytest.mark.parametrize("node", ["pve9", "pve2; reboot", "$(reboot)"])
def test_not_a_member_runs_nothing(self, node):
drv, run = self._driver()
with pytest.raises(ValueError, match="not a member"):
drv.set_node_maintenance(node, True)
run.assert_not_called()
def test_no_ssh(self):
drv, run = self._driver()
drv._node_ssh.side_effect = OSError("Authentication failed")
with pytest.raises(RuntimeError, match="SSH"):
drv.set_node_maintenance("pve2", True)
run.assert_not_called()
def test_command_refused(self):
drv, run = self._driver()
run.side_effect = RuntimeError("Command failed (exit 255): ...\nno such node 'pve2'")
with pytest.raises(RuntimeError, match="no such node"):
drv.set_node_maintenance("pve2", True)