Files
napalm-proxmox/napalm_proxmox/vm_migrate_mixin.py
T
Christian Manivong 0704d9a102
driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
feat: HA state, backups and votes in get_clusters(); migrate_vm, preflight, maintenance
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool
members, the newest backup per guest on each backup storage (PBS included) and
/cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a
guest of the connected node and follows the task (HA-managed guests through
the HA manager), raising with the task log's ERROR lines;
get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance()
runs ha-manager over SSH.

Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
2026-11-01 22:15:01 +01:00

256 lines
11 KiB
Python

"""HypervisorDriver migration methods for Proxmox VE: preflight, migrate, HA maintenance.
A guest is addressed like every other VM method of this driver: by vmid or name,
on the node the driver is connected to (``_resolve_vm``).
- ``get_vm_migration_preflight``: a VM asks ``GET .../qemu/{vmid}/migrate``; a
container has no such endpoint, so the online members are its candidates.
A member the cluster calls offline is refused (``"node offline"``) for both,
also where the VM endpoint would list it as allowed.
- ``migrate_vm``: ``POST .../{qemu|lxc}/{vmid}/migrate`` and the task is followed
to its end; a failure raises with the task log's ``ERROR:`` lines. A guest HA
manages migrates through a ``hamigrate`` task that only hands the request to
the HA manager; the driver then follows ``/cluster/ha/status/current`` until
the guest is on the target, and on a failure reads the migration task's log.
- ``set_node_maintenance``: PVE has no REST endpoint for it; it runs
``ha-manager crm-command node-maintenance`` over the driver's SSH session.
The HA manager moves the node's HA guests away after the command returned.
"""
from __future__ import annotations
import logging
import shlex
from time import monotonic, sleep
from typing import TYPE_CHECKING, Any
from napalm_proxmox.migration import lxc_local, qemu_local, refusal, task_errors, upid_field
from napalm_proxmox.pve_values import to_flag
if TYPE_CHECKING:
# Type-only: newer than the napalm_device_types floor in pyproject.toml.
from napalm_device_types import MigrationPreflightDict
logger = logging.getLogger(__name__)
_JsonDict = dict[str, Any]
#: Enough to reach the ERROR lines at the end of a long storage migration's log.
_TASK_LOG_LINES = 50000
_HA_POLL_SECONDS = 5
#: CRM states while the HA manager is moving a guest.
_HA_MOVING = frozenset({"migrate", "relocate"})
class ProxmoxVMMigrateMixin:
"""Relies on the driver, ``_resolve_vm``/``_guest_api``, ``_members`` and
``_wait_for_task``/``_run_node_command``/``_node_ssh``."""
_api: Any
_node_name: str
hostname: str
timeout: int
_node_api: Any
_resolve_vm: Any
_guest_api: Any
_members: Any
_wait_for_task: Any
_node_ssh: Any
_run_node_command: Any
# -- cluster membership -------------------------------------------------------
def _cluster_members(self) -> list[Any]:
status = self._api.cluster.status.get() or []
clustered = any(s.get("type") == "cluster" for s in status)
return list(self._members(status if clustered else []))
def _other_member(self, node: str) -> None:
names = [m.name for m in self._cluster_members()]
if node == self._node_name or node not in names:
raise ValueError(
f"{node!r} is not another member of the cluster of {self._node_name!r} "
f"(members: {', '.join(names)})"
)
def _storage_sharing(self) -> dict[str, bool]:
rows = self._api.cluster.resources.get(type="storage") or []
sharing: dict[str, bool] = {}
for row in rows:
if row.get("type") == "storage" and row.get("storage"):
name = str(row["storage"])
sharing[name] = sharing.get(name, False) or to_flag(row.get("shared"))
return sharing
# -- preflight ----------------------------------------------------------------
def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict:
"""Where the guest could move to within its cluster, and what blocks it."""
vmid, vm_type = self._resolve_vm(name)
others = [m for m in self._cluster_members() if m.name != self._node_name]
offline = [m.name for m in others if not m.online]
if vm_type == "vm":
return self._qemu_preflight(vmid, offline)
return self._lxc_preflight(vmid, [m.name for m in others if m.online], offline)
def _qemu_preflight(self, vmid: int, offline: list[str]) -> MigrationPreflightDict:
answer = self._guest_api(vmid, "vm").migrate.get() or {}
refused = {
str(node): refusal(blockers)
for node, blockers in (answer.get("not_allowed_nodes") or {}).items()
}
for node in offline:
refused[node] = "; ".join(filter(None, ["node offline", refused.get(node)]))
disks, resources = qemu_local(answer)
return {
"running": to_flag(answer.get("running")),
"allowed_nodes": [
str(n) for n in answer.get("allowed_nodes") or [] if n not in offline
],
"refused_nodes": refused,
"local_disks": disks,
"local_resources": resources,
}
def _lxc_running(self, vmid: int) -> bool:
status = self._guest_api(vmid, "container").status.current.get() or {}
return bool(status.get("status") == "running")
def _lxc_preflight(
self, vmid: int, online: list[str], offline: list[str]
) -> MigrationPreflightDict:
config = self._guest_api(vmid, "container").config.get() or {}
disks, resources = lxc_local(config, self._storage_sharing())
return {
"running": self._lxc_running(vmid),
"allowed_nodes": online,
"refused_nodes": {n: "node offline" for n in offline},
"local_disks": disks,
"local_resources": resources,
}
# -- migration ----------------------------------------------------------------
def migrate_vm(
self,
name: str,
target_node: str,
online: bool = True,
target_storage: str | None = None,
timeout: int = 3600,
) -> None:
"""Move the guest to *target_node*; block until Proxmox VE finished."""
vmid, vm_type = self._resolve_vm(name)
self._other_member(target_node)
deadline = monotonic() + timeout
params = self._qemu_params(vmid, online) if vm_type == "vm" else self._lxc_params(vmid)
if target_storage:
params["targetstorage" if vm_type == "vm" else "target-storage"] = target_storage
try:
upid = str(self._guest_api(vmid, vm_type).migrate.post(target=target_node, **params))
except Exception as exc:
raise RuntimeError(
f"Migration of {vm_type} {vmid} to {target_node!r} refused: {exc}"
) from exc
self._follow_task(upid, vmid, timeout)
if upid_field(upid, 5) == "hamigrate":
self._follow_ha(vmid, vm_type, target_node, deadline, upid)
def _qemu_params(self, vmid: int, online: bool) -> dict[str, Any]:
answer = self._guest_api(vmid, "vm").migrate.get() or {}
params: dict[str, Any] = {}
if online and to_flag(answer.get("running")):
params["online"] = 1
if qemu_local(answer)[0]:
params["with-local-disks"] = 1
return params
def _lxc_params(self, vmid: int) -> dict[str, Any]:
# A container cannot move while running; restart mode stops and restarts it.
return {"restart": 1} if self._lxc_running(vmid) else {}
def _task_errors(self, upid: str) -> list[str]:
try:
log = self._node_api().tasks(upid).log.get(limit=_TASK_LOG_LINES) or []
except Exception as exc:
logger.debug("Cannot read the log of task %s: %s", upid, exc)
return []
return task_errors(log)
def _follow_task(self, upid: str, vmid: int, timeout: int) -> None:
try:
self._wait_for_task(upid, timeout=timeout)
except RuntimeError as exc:
errors = self._task_errors(upid)
if not errors:
raise
raise RuntimeError(f"Migration of guest {vmid} failed: {'; '.join(errors)}") from exc
# -- a guest HA manages -------------------------------------------------------
def _ha_state(self, sid: str) -> tuple[str | None, str | None]:
rows = self._api.cluster.ha.status.current.get() or []
service: _JsonDict = next((r for r in rows if r.get("sid") == sid), {})
return service.get("node"), service.get("crm_state") or service.get("state")
def _follow_ha(
self, vmid: int, vm_type: str, target: str, deadline: float, ha_upid: str
) -> None:
sid = f"{'vm' if vm_type == 'vm' else 'ct'}:{vmid}"
moving_seen = False
while True:
node, state = self._ha_state(sid)
if node == target and state not in _HA_MOVING:
return
if state == "error":
self._ha_failed(vmid, vm_type, ha_upid, f"HA manager put {sid} into error state")
if state in _HA_MOVING:
moving_seen = True
elif moving_seen:
self._ha_failed(
vmid, vm_type, ha_upid, f"HA manager left {sid} on {node} ({state})"
)
if monotonic() > deadline:
raise RuntimeError(
f"Migration of {sid} to {target!r} timed out on {node} ({state})"
)
sleep(_HA_POLL_SECONDS)
def _ha_failed(self, vmid: int, vm_type: str, ha_upid: str, what: str) -> None:
"""Raise *what* with the ERROR lines of the migration the HA manager ran."""
errors: list[str] = []
kind = "qmigrate" if vm_type == "vm" else "vzmigrate"
try:
tasks = self._node_api().tasks.get(vmid=vmid, typefilter=kind, limit=1) or []
since = int(upid_field(ha_upid, 4) or "0", 16)
if tasks and int(tasks[0].get("starttime") or 0) >= since:
errors = self._task_errors(str(tasks[0].get("upid")))
except Exception as exc:
logger.debug("Cannot find the migration task of guest %s: %s", vmid, exc)
raise RuntimeError(f"{what}: {'; '.join(errors)}" if errors else what)
# -- HA maintenance -----------------------------------------------------------
def set_node_maintenance(self, node: str, enabled: bool) -> None:
"""Switch HA maintenance mode of a cluster node, over SSH (no REST endpoint)."""
names = [m.name for m in self._cluster_members()]
if node not in names:
raise ValueError(
f"{node!r} is not a member of the cluster (members: {', '.join(names)})"
)
action = "enable" if enabled else "disable"
try:
self._node_ssh()
except Exception as exc:
raise RuntimeError(
f"Cannot {action} HA maintenance of node {node!r}: SSH to {self.hostname} "
f"is not available ({exc})"
) from exc
command = f"ha-manager crm-command node-maintenance {action} {shlex.quote(node)}"
try:
self._run_node_command(command, timeout=self.timeout)
except Exception as exc:
raise RuntimeError(
f"ha-manager refused to {action} maintenance of {node!r}: {exc}"
) from exc