driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool members, the newest backup per guest on each backup storage (PBS included) and /cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a guest of the connected node and follows the task (HA-managed guests through the HA manager), raising with the task log's ERROR lines; get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance() runs ha-manager over SSH. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
256 lines
11 KiB
Python
256 lines
11 KiB
Python
"""HypervisorDriver migration methods for Proxmox VE: preflight, migrate, HA maintenance.
|
|
|
|
A guest is addressed like every other VM method of this driver: by vmid or name,
|
|
on the node the driver is connected to (``_resolve_vm``).
|
|
|
|
- ``get_vm_migration_preflight``: a VM asks ``GET .../qemu/{vmid}/migrate``; a
|
|
container has no such endpoint, so the online members are its candidates.
|
|
A member the cluster calls offline is refused (``"node offline"``) for both,
|
|
also where the VM endpoint would list it as allowed.
|
|
- ``migrate_vm``: ``POST .../{qemu|lxc}/{vmid}/migrate`` and the task is followed
|
|
to its end; a failure raises with the task log's ``ERROR:`` lines. A guest HA
|
|
manages migrates through a ``hamigrate`` task that only hands the request to
|
|
the HA manager; the driver then follows ``/cluster/ha/status/current`` until
|
|
the guest is on the target, and on a failure reads the migration task's log.
|
|
- ``set_node_maintenance``: PVE has no REST endpoint for it; it runs
|
|
``ha-manager crm-command node-maintenance`` over the driver's SSH session.
|
|
The HA manager moves the node's HA guests away after the command returned.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import shlex
|
|
from time import monotonic, sleep
|
|
from typing import TYPE_CHECKING, Any
|
|
|
|
from napalm_proxmox.migration import lxc_local, qemu_local, refusal, task_errors, upid_field
|
|
from napalm_proxmox.pve_values import to_flag
|
|
|
|
if TYPE_CHECKING:
|
|
# Type-only: newer than the napalm_device_types floor in pyproject.toml.
|
|
from napalm_device_types import MigrationPreflightDict
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_JsonDict = dict[str, Any]
|
|
|
|
#: Enough to reach the ERROR lines at the end of a long storage migration's log.
|
|
_TASK_LOG_LINES = 50000
|
|
_HA_POLL_SECONDS = 5
|
|
#: CRM states while the HA manager is moving a guest.
|
|
_HA_MOVING = frozenset({"migrate", "relocate"})
|
|
|
|
|
|
class ProxmoxVMMigrateMixin:
|
|
"""Relies on the driver, ``_resolve_vm``/``_guest_api``, ``_members`` and
|
|
``_wait_for_task``/``_run_node_command``/``_node_ssh``."""
|
|
|
|
_api: Any
|
|
_node_name: str
|
|
hostname: str
|
|
timeout: int
|
|
_node_api: Any
|
|
_resolve_vm: Any
|
|
_guest_api: Any
|
|
_members: Any
|
|
_wait_for_task: Any
|
|
_node_ssh: Any
|
|
_run_node_command: Any
|
|
|
|
# -- cluster membership -------------------------------------------------------
|
|
|
|
def _cluster_members(self) -> list[Any]:
|
|
status = self._api.cluster.status.get() or []
|
|
clustered = any(s.get("type") == "cluster" for s in status)
|
|
return list(self._members(status if clustered else []))
|
|
|
|
def _other_member(self, node: str) -> None:
|
|
names = [m.name for m in self._cluster_members()]
|
|
if node == self._node_name or node not in names:
|
|
raise ValueError(
|
|
f"{node!r} is not another member of the cluster of {self._node_name!r} "
|
|
f"(members: {', '.join(names)})"
|
|
)
|
|
|
|
def _storage_sharing(self) -> dict[str, bool]:
|
|
rows = self._api.cluster.resources.get(type="storage") or []
|
|
sharing: dict[str, bool] = {}
|
|
for row in rows:
|
|
if row.get("type") == "storage" and row.get("storage"):
|
|
name = str(row["storage"])
|
|
sharing[name] = sharing.get(name, False) or to_flag(row.get("shared"))
|
|
return sharing
|
|
|
|
# -- preflight ----------------------------------------------------------------
|
|
|
|
def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict:
|
|
"""Where the guest could move to within its cluster, and what blocks it."""
|
|
vmid, vm_type = self._resolve_vm(name)
|
|
others = [m for m in self._cluster_members() if m.name != self._node_name]
|
|
offline = [m.name for m in others if not m.online]
|
|
if vm_type == "vm":
|
|
return self._qemu_preflight(vmid, offline)
|
|
return self._lxc_preflight(vmid, [m.name for m in others if m.online], offline)
|
|
|
|
def _qemu_preflight(self, vmid: int, offline: list[str]) -> MigrationPreflightDict:
|
|
answer = self._guest_api(vmid, "vm").migrate.get() or {}
|
|
refused = {
|
|
str(node): refusal(blockers)
|
|
for node, blockers in (answer.get("not_allowed_nodes") or {}).items()
|
|
}
|
|
for node in offline:
|
|
refused[node] = "; ".join(filter(None, ["node offline", refused.get(node)]))
|
|
disks, resources = qemu_local(answer)
|
|
return {
|
|
"running": to_flag(answer.get("running")),
|
|
"allowed_nodes": [
|
|
str(n) for n in answer.get("allowed_nodes") or [] if n not in offline
|
|
],
|
|
"refused_nodes": refused,
|
|
"local_disks": disks,
|
|
"local_resources": resources,
|
|
}
|
|
|
|
def _lxc_running(self, vmid: int) -> bool:
|
|
status = self._guest_api(vmid, "container").status.current.get() or {}
|
|
return bool(status.get("status") == "running")
|
|
|
|
def _lxc_preflight(
|
|
self, vmid: int, online: list[str], offline: list[str]
|
|
) -> MigrationPreflightDict:
|
|
config = self._guest_api(vmid, "container").config.get() or {}
|
|
disks, resources = lxc_local(config, self._storage_sharing())
|
|
return {
|
|
"running": self._lxc_running(vmid),
|
|
"allowed_nodes": online,
|
|
"refused_nodes": {n: "node offline" for n in offline},
|
|
"local_disks": disks,
|
|
"local_resources": resources,
|
|
}
|
|
|
|
# -- migration ----------------------------------------------------------------
|
|
|
|
def migrate_vm(
|
|
self,
|
|
name: str,
|
|
target_node: str,
|
|
online: bool = True,
|
|
target_storage: str | None = None,
|
|
timeout: int = 3600,
|
|
) -> None:
|
|
"""Move the guest to *target_node*; block until Proxmox VE finished."""
|
|
vmid, vm_type = self._resolve_vm(name)
|
|
self._other_member(target_node)
|
|
deadline = monotonic() + timeout
|
|
params = self._qemu_params(vmid, online) if vm_type == "vm" else self._lxc_params(vmid)
|
|
if target_storage:
|
|
params["targetstorage" if vm_type == "vm" else "target-storage"] = target_storage
|
|
try:
|
|
upid = str(self._guest_api(vmid, vm_type).migrate.post(target=target_node, **params))
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"Migration of {vm_type} {vmid} to {target_node!r} refused: {exc}"
|
|
) from exc
|
|
self._follow_task(upid, vmid, timeout)
|
|
if upid_field(upid, 5) == "hamigrate":
|
|
self._follow_ha(vmid, vm_type, target_node, deadline, upid)
|
|
|
|
def _qemu_params(self, vmid: int, online: bool) -> dict[str, Any]:
|
|
answer = self._guest_api(vmid, "vm").migrate.get() or {}
|
|
params: dict[str, Any] = {}
|
|
if online and to_flag(answer.get("running")):
|
|
params["online"] = 1
|
|
if qemu_local(answer)[0]:
|
|
params["with-local-disks"] = 1
|
|
return params
|
|
|
|
def _lxc_params(self, vmid: int) -> dict[str, Any]:
|
|
# A container cannot move while running; restart mode stops and restarts it.
|
|
return {"restart": 1} if self._lxc_running(vmid) else {}
|
|
|
|
def _task_errors(self, upid: str) -> list[str]:
|
|
try:
|
|
log = self._node_api().tasks(upid).log.get(limit=_TASK_LOG_LINES) or []
|
|
except Exception as exc:
|
|
logger.debug("Cannot read the log of task %s: %s", upid, exc)
|
|
return []
|
|
return task_errors(log)
|
|
|
|
def _follow_task(self, upid: str, vmid: int, timeout: int) -> None:
|
|
try:
|
|
self._wait_for_task(upid, timeout=timeout)
|
|
except RuntimeError as exc:
|
|
errors = self._task_errors(upid)
|
|
if not errors:
|
|
raise
|
|
raise RuntimeError(f"Migration of guest {vmid} failed: {'; '.join(errors)}") from exc
|
|
|
|
# -- a guest HA manages -------------------------------------------------------
|
|
|
|
def _ha_state(self, sid: str) -> tuple[str | None, str | None]:
|
|
rows = self._api.cluster.ha.status.current.get() or []
|
|
service: _JsonDict = next((r for r in rows if r.get("sid") == sid), {})
|
|
return service.get("node"), service.get("crm_state") or service.get("state")
|
|
|
|
def _follow_ha(
|
|
self, vmid: int, vm_type: str, target: str, deadline: float, ha_upid: str
|
|
) -> None:
|
|
sid = f"{'vm' if vm_type == 'vm' else 'ct'}:{vmid}"
|
|
moving_seen = False
|
|
while True:
|
|
node, state = self._ha_state(sid)
|
|
if node == target and state not in _HA_MOVING:
|
|
return
|
|
if state == "error":
|
|
self._ha_failed(vmid, vm_type, ha_upid, f"HA manager put {sid} into error state")
|
|
if state in _HA_MOVING:
|
|
moving_seen = True
|
|
elif moving_seen:
|
|
self._ha_failed(
|
|
vmid, vm_type, ha_upid, f"HA manager left {sid} on {node} ({state})"
|
|
)
|
|
if monotonic() > deadline:
|
|
raise RuntimeError(
|
|
f"Migration of {sid} to {target!r} timed out on {node} ({state})"
|
|
)
|
|
sleep(_HA_POLL_SECONDS)
|
|
|
|
def _ha_failed(self, vmid: int, vm_type: str, ha_upid: str, what: str) -> None:
|
|
"""Raise *what* with the ERROR lines of the migration the HA manager ran."""
|
|
errors: list[str] = []
|
|
kind = "qmigrate" if vm_type == "vm" else "vzmigrate"
|
|
try:
|
|
tasks = self._node_api().tasks.get(vmid=vmid, typefilter=kind, limit=1) or []
|
|
since = int(upid_field(ha_upid, 4) or "0", 16)
|
|
if tasks and int(tasks[0].get("starttime") or 0) >= since:
|
|
errors = self._task_errors(str(tasks[0].get("upid")))
|
|
except Exception as exc:
|
|
logger.debug("Cannot find the migration task of guest %s: %s", vmid, exc)
|
|
raise RuntimeError(f"{what}: {'; '.join(errors)}" if errors else what)
|
|
|
|
# -- HA maintenance -----------------------------------------------------------
|
|
|
|
def set_node_maintenance(self, node: str, enabled: bool) -> None:
|
|
"""Switch HA maintenance mode of a cluster node, over SSH (no REST endpoint)."""
|
|
names = [m.name for m in self._cluster_members()]
|
|
if node not in names:
|
|
raise ValueError(
|
|
f"{node!r} is not a member of the cluster (members: {', '.join(names)})"
|
|
)
|
|
action = "enable" if enabled else "disable"
|
|
try:
|
|
self._node_ssh()
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"Cannot {action} HA maintenance of node {node!r}: SSH to {self.hostname} "
|
|
f"is not available ({exc})"
|
|
) from exc
|
|
command = f"ha-manager crm-command node-maintenance {action} {shlex.quote(node)}"
|
|
try:
|
|
self._run_node_command(command, timeout=self.timeout)
|
|
except Exception as exc:
|
|
raise RuntimeError(
|
|
f"ha-manager refused to {action} maintenance of {node!r}: {exc}"
|
|
) from exc
|