driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool members, the newest backup per guest on each backup storage (PBS included) and /cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a guest of the connected node and follows the task (HA-managed guests through the HA manager), raising with the task log's ERROR lines; get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance() runs ha-manager over SSH. Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
383 lines
15 KiB
Python
383 lines
15 KiB
Python
"""Moving a guest to another node: preflight, migration, HA maintenance mode.
|
|
|
|
Payloads: ``GET /nodes/{node}/qemu/{vmid}/migrate`` follows qemu-server's
|
|
``migrate_vm_precondition`` (PVE 9: ``not_allowed_nodes`` may name
|
|
``unavailable_storages``, ``unavailable-resources`` and
|
|
``blocking-ha-resources``); the task log follows ``/nodes/{node}/tasks/{upid}/log``
|
|
(``n``/``t`` rows). A guest that HA manages migrates through a ``hamigrate``
|
|
task that only hands the request to the HA manager.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from tests.test_clusters import CLUSTER_RESOURCES, CLUSTER_STATUS, GIB, _api, _driver, _storage_row
|
|
|
|
QEMU_LIST = [
|
|
{"vmid": 100, "name": "web", "status": "running", "maxmem": 4 * GIB},
|
|
{"vmid": 102, "name": "idle", "status": "stopped", "maxmem": 2 * GIB},
|
|
]
|
|
LXC_LIST = [{"vmid": "101", "name": "dns", "status": "running", "type": "lxc"}]
|
|
|
|
PREFLIGHT_RUNNING = {
|
|
"running": 1,
|
|
"allowed_nodes": ["pve2", "pve3"],
|
|
"not_allowed_nodes": {},
|
|
"local_disks": [
|
|
{
|
|
"volid": "local-lvm:vm-100-disk-0",
|
|
"size": 32 * GIB,
|
|
"drivename": "scsi0",
|
|
"is_attached": 1,
|
|
"referenced_in_config": 1,
|
|
},
|
|
{"volid": "local:iso/debian-13.iso", "cdrom": 1, "size": 700 * 1024**2},
|
|
# Regenerated on the target: neither a disk to copy nor a blocker.
|
|
{"volid": "local-lvm:vm-100-cloudinit", "cdrom": 1, "size": 4 * 1024**2},
|
|
],
|
|
"local_resources": [],
|
|
"mapped-resources": [],
|
|
"mapped-resource-info": {},
|
|
"has-dbus-vmstate": 1,
|
|
}
|
|
|
|
PREFLIGHT_BLOCKED = {
|
|
"running": 0,
|
|
"allowed_nodes": [],
|
|
"not_allowed_nodes": {
|
|
"pve2": {"unavailable_storages": ["local-zfs"]},
|
|
"pve3": {
|
|
"blocking-ha-resources": [{"sid": "vm:101", "cause": "resource-affinity"}],
|
|
"unavailable-resources": ["gpu-a"],
|
|
},
|
|
},
|
|
"local_disks": [],
|
|
"local_resources": ["hostpci0", "usb0"],
|
|
"mapped-resources": ["gpu-a"],
|
|
"mapped-resource-info": {"gpu-a": {"live-migration": 0}},
|
|
"has-dbus-vmstate": 1,
|
|
}
|
|
|
|
LXC_CONFIG = {
|
|
"hostname": "dns",
|
|
"rootfs": "local-lvm:vm-101-disk-0,size=8G",
|
|
"mp0": "ceph-vm:vm-101-disk-1,mp=/data,size=32G",
|
|
"mp1": "/srv/share,mp=/share",
|
|
"mp2": "/mnt/nfs,mp=/nfs,shared=1",
|
|
"unused0": "local-lvm:vm-101-disk-2",
|
|
"net0": "name=eth0,bridge=vmbr0,ip=dhcp",
|
|
"digest": "abc",
|
|
}
|
|
|
|
UPID_QEMU = "UPID:pve1:000ABCDE:0123ABCD:672AB000:qmigrate:100:root@pam:"
|
|
UPID_LXC = "UPID:pve1:000ABCDF:0123ABCE:672AB001:vzmigrate:101:root@pam:"
|
|
UPID_HA = "UPID:pve1:000ABCE0:0123ABCF:672AB002:hamigrate:100:root@pam:"
|
|
|
|
FAILED_LOG = [
|
|
{"n": 1, "t": "2026-11-01 10:00:00 starting migration of VM 100 to node 'pve2' (10.0.0.2)"},
|
|
{"n": 2, "t": "2026-11-01 10:00:01 starting VM 100 on remote node 'pve2'"},
|
|
{
|
|
"n": 3,
|
|
"t": "2026-11-01 10:00:02 ERROR: online migrate failure - remote command failed "
|
|
"with exit code 255",
|
|
},
|
|
{"n": 4, "t": "2026-11-01 10:00:02 aborting phase 2 - cleanup resources"},
|
|
{
|
|
"n": 5,
|
|
"t": "2026-11-01 10:00:03 ERROR: migration finished with problems (duration 00:00:03)",
|
|
},
|
|
{"n": 6, "t": "TASK ERROR: migration problems"},
|
|
]
|
|
|
|
|
|
def _migration_api(
|
|
preflight: dict[str, Any] | None = None,
|
|
task: dict[str, Any] | None = None,
|
|
log: list[dict[str, Any]] | None = None,
|
|
status: list[dict[str, Any]] | None = None,
|
|
) -> tuple[MagicMock, MagicMock]:
|
|
resources = [
|
|
*CLUSTER_RESOURCES,
|
|
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB),
|
|
]
|
|
api = _api(cluster_status=status, resources=resources)
|
|
node = api.nodes("pve1")
|
|
node.qemu.get.return_value = QEMU_LIST
|
|
node.lxc.get.return_value = LXC_LIST
|
|
node.qemu.return_value.migrate.get.return_value = preflight or PREFLIGHT_RUNNING
|
|
node.qemu.return_value.migrate.post.return_value = UPID_QEMU
|
|
node.lxc.return_value.migrate.post.return_value = UPID_LXC
|
|
node.lxc.return_value.status.current.get.return_value = {"status": "running", "vmid": 101}
|
|
node.lxc.return_value.config.get.return_value = LXC_CONFIG
|
|
node.tasks.return_value.status.get.return_value = task or {
|
|
"status": "stopped",
|
|
"exitstatus": "OK",
|
|
}
|
|
node.tasks.return_value.log.get.return_value = log or []
|
|
return api, node
|
|
|
|
|
|
class TestPreflightQemu:
|
|
def test_allowed_nodes_and_local_disks(self):
|
|
api, node = _migration_api()
|
|
result = _driver(api).get_vm_migration_preflight("web")
|
|
assert result == {
|
|
"running": True,
|
|
# pve3 is offline: the platform would allow it, it cannot take the VM now.
|
|
"allowed_nodes": ["pve2"],
|
|
"refused_nodes": {"pve3": "node offline"},
|
|
"local_disks": ["local-lvm:vm-100-disk-0"],
|
|
# A local ISO in a CD drive stops the migration.
|
|
"local_resources": ["cdrom=local:iso/debian-13.iso"],
|
|
}
|
|
node.qemu.assert_any_call(100)
|
|
|
|
def test_refusal_reasons_in_plain_words(self):
|
|
all_online = [dict(s, online=1) if s["type"] == "node" else s for s in CLUSTER_STATUS]
|
|
api, _ = _migration_api(preflight=PREFLIGHT_BLOCKED, status=all_online)
|
|
result = _driver(api).get_vm_migration_preflight("100")
|
|
assert result["running"] is False
|
|
assert result["allowed_nodes"] == []
|
|
assert result["refused_nodes"] == {
|
|
"pve2": "storage local-zfs is not available there",
|
|
"pve3": "HA resource vm:101 blocks it (resource-affinity rule); "
|
|
"mapped resource gpu-a is not available there",
|
|
}
|
|
assert result["local_resources"] == ["hostpci0", "usb0"]
|
|
|
|
def test_unknown_reason_is_passed_on(self):
|
|
preflight = {
|
|
"running": 0,
|
|
"allowed_nodes": [],
|
|
"not_allowed_nodes": {"pve2": {"some-future-check": ["x", "y"]}},
|
|
"local_disks": [],
|
|
"local_resources": [],
|
|
}
|
|
api, _ = _migration_api(preflight=preflight)
|
|
refused = _driver(api).get_vm_migration_preflight("web")["refused_nodes"]
|
|
assert refused["pve2"] == "some-future-check: x, y"
|
|
|
|
def test_unknown_guest(self):
|
|
api, _ = _migration_api()
|
|
with pytest.raises(ValueError, match="nope"):
|
|
_driver(api).get_vm_migration_preflight("nope")
|
|
|
|
|
|
class TestPreflightLxc:
|
|
def test_members_and_local_volumes(self):
|
|
api, node = _migration_api()
|
|
result = _driver(api).get_vm_migration_preflight("dns")
|
|
assert result == {
|
|
"running": True,
|
|
"allowed_nodes": ["pve2"],
|
|
"refused_nodes": {"pve3": "node offline"},
|
|
# ceph-vm is shared; local-lvm is not.
|
|
"local_disks": ["local-lvm:vm-101-disk-0", "local-lvm:vm-101-disk-2"],
|
|
# A bind mount that is not marked shared stops the migration.
|
|
"local_resources": ["mp1"],
|
|
}
|
|
node.qemu.return_value.migrate.get.assert_not_called()
|
|
|
|
def test_stopped_container(self):
|
|
api, node = _migration_api()
|
|
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
|
|
assert _driver(api).get_vm_migration_preflight("101")["running"] is False
|
|
|
|
|
|
class TestMigrateQemu:
|
|
def test_running_vm_moves_online_with_its_local_disks(self):
|
|
api, node = _migration_api()
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
node.qemu.return_value.migrate.post.assert_called_once_with(
|
|
target="pve2", online=1, **{"with-local-disks": 1}
|
|
)
|
|
node.tasks.assert_called_with(UPID_QEMU)
|
|
|
|
def test_target_storage_and_offline(self):
|
|
preflight = dict(PREFLIGHT_RUNNING, local_disks=[])
|
|
api, node = _migration_api(preflight=preflight)
|
|
_driver(api).migrate_vm("web", "pve2", online=False, target_storage="ceph-vm")
|
|
node.qemu.return_value.migrate.post.assert_called_once_with(
|
|
target="pve2", targetstorage="ceph-vm"
|
|
)
|
|
|
|
def test_stopped_vm_is_not_moved_online(self):
|
|
preflight = dict(PREFLIGHT_RUNNING, running=0, local_disks=[])
|
|
api, node = _migration_api(preflight=preflight)
|
|
_driver(api).migrate_vm("idle", "pve2")
|
|
node.qemu.return_value.migrate.post.assert_called_once_with(target="pve2")
|
|
|
|
def test_failure_raises_with_the_task_logs_errors(self):
|
|
api, node = _migration_api(
|
|
task={"status": "stopped", "exitstatus": "migration problems"}, log=FAILED_LOG
|
|
)
|
|
with pytest.raises(RuntimeError) as exc:
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
message = str(exc.value)
|
|
assert "online migrate failure - remote command failed with exit code 255" in message
|
|
assert "migration finished with problems (duration 00:00:03)" in message
|
|
assert "starting VM 100" not in message
|
|
node.tasks.return_value.log.get.assert_called_once()
|
|
|
|
def test_failure_without_error_lines_keeps_the_task_status(self):
|
|
api, _ = _migration_api(
|
|
task={"status": "stopped", "exitstatus": "migration aborted"},
|
|
log=[{"n": 1, "t": "starting migration"}],
|
|
)
|
|
with pytest.raises(RuntimeError, match="migration aborted"):
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
|
|
def test_refused_request_raises(self):
|
|
api, node = _migration_api()
|
|
node.qemu.return_value.migrate.post.side_effect = RuntimeError(
|
|
"500 Internal Server Error: VM is locked (backup)"
|
|
)
|
|
with pytest.raises(RuntimeError, match="VM is locked"):
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
|
|
|
|
class TestMigrateLxc:
|
|
def test_running_container_moves_in_restart_mode(self):
|
|
api, node = _migration_api()
|
|
_driver(api).migrate_vm("dns", "pve2", target_storage="local-lvm")
|
|
node.lxc.return_value.migrate.post.assert_called_once_with(
|
|
target="pve2", restart=1, **{"target-storage": "local-lvm"}
|
|
)
|
|
node.tasks.assert_called_with(UPID_LXC)
|
|
|
|
def test_stopped_container(self):
|
|
api, node = _migration_api()
|
|
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
|
|
_driver(api).migrate_vm("dns", "pve2")
|
|
node.lxc.return_value.migrate.post.assert_called_once_with(target="pve2")
|
|
|
|
|
|
class TestTargetRefusedBeforeTheApi:
|
|
@pytest.mark.parametrize("target", ["pve1", "pve9", "pve2; reboot"])
|
|
def test_not_another_member(self, target):
|
|
api, node = _migration_api()
|
|
with pytest.raises(ValueError, match="not another member"):
|
|
_driver(api).migrate_vm("web", target)
|
|
node.qemu.return_value.migrate.post.assert_not_called()
|
|
|
|
|
|
def _ha_status(node: str, crm_state: str) -> list[dict[str, Any]]:
|
|
return [
|
|
{"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"},
|
|
{
|
|
"id": "service:vm:100",
|
|
"type": "service",
|
|
"sid": "vm:100",
|
|
"node": node,
|
|
"crm_state": crm_state,
|
|
"state": crm_state,
|
|
"status": f"vm:100 ({node}, {crm_state})",
|
|
"request_state": "started",
|
|
},
|
|
]
|
|
|
|
|
|
class TestMigrateHaManaged:
|
|
"""PVE hands an HA guest's migration to the HA manager: the hamigrate task only
|
|
queues the request, so the driver follows the HA manager until the guest is there."""
|
|
|
|
def _api(self, states: list[list[dict[str, Any]]]) -> tuple[MagicMock, MagicMock]:
|
|
api, node = _migration_api()
|
|
node.qemu.return_value.migrate.post.return_value = UPID_HA
|
|
api.cluster.ha.status.current.get.side_effect = states
|
|
return api, node
|
|
|
|
def test_waits_until_the_guest_runs_on_the_target(self):
|
|
api, _ = self._api(
|
|
[
|
|
_ha_status("pve1", "started"),
|
|
_ha_status("pve1", "migrate"),
|
|
_ha_status("pve2", "started"),
|
|
]
|
|
)
|
|
with patch("napalm_proxmox.vm_migrate_mixin.sleep") as sleep:
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
assert api.cluster.ha.status.current.get.call_count == 3
|
|
assert sleep.call_count == 2
|
|
|
|
def test_back_on_the_source_node_is_a_failure_with_the_migration_log(self):
|
|
api, node = self._api([_ha_status("pve1", "migrate"), _ha_status("pve1", "started")])
|
|
node.tasks.get.return_value = [
|
|
{
|
|
"upid": UPID_QEMU,
|
|
"type": "qmigrate",
|
|
"id": "100",
|
|
"starttime": 0x672AB002 + 12,
|
|
"status": "migration problems",
|
|
}
|
|
]
|
|
node.tasks.return_value.log.get.return_value = FAILED_LOG
|
|
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
|
|
with pytest.raises(RuntimeError, match="online migrate failure"):
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
node.tasks.get.assert_called_once_with(vmid=100, typefilter="qmigrate", limit=1)
|
|
|
|
def test_error_state_raises(self):
|
|
api, node = self._api([_ha_status("pve1", "error")])
|
|
node.tasks.get.return_value = []
|
|
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
|
|
with pytest.raises(RuntimeError, match="error"):
|
|
_driver(api).migrate_vm("web", "pve2")
|
|
|
|
def test_times_out(self):
|
|
api, _ = self._api([_ha_status("pve1", "migrate")] * 50)
|
|
clock = iter(range(0, 10000, 30))
|
|
with (
|
|
patch("napalm_proxmox.vm_migrate_mixin.sleep"),
|
|
patch("napalm_proxmox.vm_migrate_mixin.monotonic", lambda: next(clock)),
|
|
):
|
|
with pytest.raises(RuntimeError, match="timed out"):
|
|
_driver(api).migrate_vm("web", "pve2", timeout=120)
|
|
|
|
|
|
class TestNodeMaintenance:
|
|
def _driver(self) -> tuple[Any, MagicMock]:
|
|
api, _ = _migration_api()
|
|
drv = _driver(api)
|
|
drv._node_ssh = MagicMock() # type: ignore[method-assign]
|
|
drv._run_node_command = MagicMock(return_value="") # type: ignore[method-assign]
|
|
return drv, drv._run_node_command
|
|
|
|
def test_enable(self):
|
|
drv, run = self._driver()
|
|
drv.set_node_maintenance("pve2", True)
|
|
run.assert_called_once_with(
|
|
"ha-manager crm-command node-maintenance enable pve2", timeout=drv.timeout
|
|
)
|
|
|
|
def test_disable(self):
|
|
drv, run = self._driver()
|
|
drv.set_node_maintenance("pve3", False)
|
|
run.assert_called_once_with(
|
|
"ha-manager crm-command node-maintenance disable pve3", timeout=drv.timeout
|
|
)
|
|
|
|
@pytest.mark.parametrize("node", ["pve9", "pve2; reboot", "$(reboot)"])
|
|
def test_not_a_member_runs_nothing(self, node):
|
|
drv, run = self._driver()
|
|
with pytest.raises(ValueError, match="not a member"):
|
|
drv.set_node_maintenance(node, True)
|
|
run.assert_not_called()
|
|
|
|
def test_no_ssh(self):
|
|
drv, run = self._driver()
|
|
drv._node_ssh.side_effect = OSError("Authentication failed")
|
|
with pytest.raises(RuntimeError, match="SSH"):
|
|
drv.set_node_maintenance("pve2", True)
|
|
run.assert_not_called()
|
|
|
|
def test_command_refused(self):
|
|
drv, run = self._driver()
|
|
run.side_effect = RuntimeError("Command failed (exit 255): ...\nno such node 'pve2'")
|
|
with pytest.raises(RuntimeError, match="no such node"):
|
|
drv.set_node_maintenance("pve2", True)
|