Files
napalm-proxmox/tests/test_vm_migration.py
T
Christian Manivong 0704d9a102
driver / standard (pull_request) Successful in 14s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 1m3s
driver / test (3.12) (pull_request) Successful in 1m10s
driver / test (3.13) (pull_request) Successful in 1m3s
driver / lowest (pull_request) Successful in 33s
driver / build (pull_request) Successful in 33s
driver / security (pull_request) Successful in 1m27s
CI / driver (pull_request) Successful in 3m4s
driver / publish (pull_request) Skipped
feat: HA state, backups and votes in get_clusters(); migrate_vm, preflight, maintenance
get_clusters() reads /cluster/ha/* (PVE 8 and 9), /cluster/backup with pool
members, the newest backup per guest on each backup storage (PBS included) and
/cluster/config/nodes votes; each read is best-effort. migrate_vm() moves a
guest of the connected node and follows the task (HA-managed guests through
the HA manager), raising with the task log's ERROR lines;
get_vm_migration_preflight() maps PVE's own pre-flight; set_node_maintenance()
runs ha-manager over SSH.

Part of NetOrk/netork#1820 (stage 7: #759, #760, #761).
2026-11-01 22:15:01 +01:00

383 lines
15 KiB
Python

"""Moving a guest to another node: preflight, migration, HA maintenance mode.
Payloads: ``GET /nodes/{node}/qemu/{vmid}/migrate`` follows qemu-server's
``migrate_vm_precondition`` (PVE 9: ``not_allowed_nodes`` may name
``unavailable_storages``, ``unavailable-resources`` and
``blocking-ha-resources``); the task log follows ``/nodes/{node}/tasks/{upid}/log``
(``n``/``t`` rows). A guest that HA manages migrates through a ``hamigrate``
task that only hands the request to the HA manager.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock, patch
import pytest
from tests.test_clusters import CLUSTER_RESOURCES, CLUSTER_STATUS, GIB, _api, _driver, _storage_row
QEMU_LIST = [
{"vmid": 100, "name": "web", "status": "running", "maxmem": 4 * GIB},
{"vmid": 102, "name": "idle", "status": "stopped", "maxmem": 2 * GIB},
]
LXC_LIST = [{"vmid": "101", "name": "dns", "status": "running", "type": "lxc"}]
PREFLIGHT_RUNNING = {
"running": 1,
"allowed_nodes": ["pve2", "pve3"],
"not_allowed_nodes": {},
"local_disks": [
{
"volid": "local-lvm:vm-100-disk-0",
"size": 32 * GIB,
"drivename": "scsi0",
"is_attached": 1,
"referenced_in_config": 1,
},
{"volid": "local:iso/debian-13.iso", "cdrom": 1, "size": 700 * 1024**2},
# Regenerated on the target: neither a disk to copy nor a blocker.
{"volid": "local-lvm:vm-100-cloudinit", "cdrom": 1, "size": 4 * 1024**2},
],
"local_resources": [],
"mapped-resources": [],
"mapped-resource-info": {},
"has-dbus-vmstate": 1,
}
PREFLIGHT_BLOCKED = {
"running": 0,
"allowed_nodes": [],
"not_allowed_nodes": {
"pve2": {"unavailable_storages": ["local-zfs"]},
"pve3": {
"blocking-ha-resources": [{"sid": "vm:101", "cause": "resource-affinity"}],
"unavailable-resources": ["gpu-a"],
},
},
"local_disks": [],
"local_resources": ["hostpci0", "usb0"],
"mapped-resources": ["gpu-a"],
"mapped-resource-info": {"gpu-a": {"live-migration": 0}},
"has-dbus-vmstate": 1,
}
LXC_CONFIG = {
"hostname": "dns",
"rootfs": "local-lvm:vm-101-disk-0,size=8G",
"mp0": "ceph-vm:vm-101-disk-1,mp=/data,size=32G",
"mp1": "/srv/share,mp=/share",
"mp2": "/mnt/nfs,mp=/nfs,shared=1",
"unused0": "local-lvm:vm-101-disk-2",
"net0": "name=eth0,bridge=vmbr0,ip=dhcp",
"digest": "abc",
}
UPID_QEMU = "UPID:pve1:000ABCDE:0123ABCD:672AB000:qmigrate:100:root@pam:"
UPID_LXC = "UPID:pve1:000ABCDF:0123ABCE:672AB001:vzmigrate:101:root@pam:"
UPID_HA = "UPID:pve1:000ABCE0:0123ABCF:672AB002:hamigrate:100:root@pam:"
FAILED_LOG = [
{"n": 1, "t": "2026-11-01 10:00:00 starting migration of VM 100 to node 'pve2' (10.0.0.2)"},
{"n": 2, "t": "2026-11-01 10:00:01 starting VM 100 on remote node 'pve2'"},
{
"n": 3,
"t": "2026-11-01 10:00:02 ERROR: online migrate failure - remote command failed "
"with exit code 255",
},
{"n": 4, "t": "2026-11-01 10:00:02 aborting phase 2 - cleanup resources"},
{
"n": 5,
"t": "2026-11-01 10:00:03 ERROR: migration finished with problems (duration 00:00:03)",
},
{"n": 6, "t": "TASK ERROR: migration problems"},
]
def _migration_api(
preflight: dict[str, Any] | None = None,
task: dict[str, Any] | None = None,
log: list[dict[str, Any]] | None = None,
status: list[dict[str, Any]] | None = None,
) -> tuple[MagicMock, MagicMock]:
resources = [
*CLUSTER_RESOURCES,
_storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB),
]
api = _api(cluster_status=status, resources=resources)
node = api.nodes("pve1")
node.qemu.get.return_value = QEMU_LIST
node.lxc.get.return_value = LXC_LIST
node.qemu.return_value.migrate.get.return_value = preflight or PREFLIGHT_RUNNING
node.qemu.return_value.migrate.post.return_value = UPID_QEMU
node.lxc.return_value.migrate.post.return_value = UPID_LXC
node.lxc.return_value.status.current.get.return_value = {"status": "running", "vmid": 101}
node.lxc.return_value.config.get.return_value = LXC_CONFIG
node.tasks.return_value.status.get.return_value = task or {
"status": "stopped",
"exitstatus": "OK",
}
node.tasks.return_value.log.get.return_value = log or []
return api, node
class TestPreflightQemu:
def test_allowed_nodes_and_local_disks(self):
api, node = _migration_api()
result = _driver(api).get_vm_migration_preflight("web")
assert result == {
"running": True,
# pve3 is offline: the platform would allow it, it cannot take the VM now.
"allowed_nodes": ["pve2"],
"refused_nodes": {"pve3": "node offline"},
"local_disks": ["local-lvm:vm-100-disk-0"],
# A local ISO in a CD drive stops the migration.
"local_resources": ["cdrom=local:iso/debian-13.iso"],
}
node.qemu.assert_any_call(100)
def test_refusal_reasons_in_plain_words(self):
all_online = [dict(s, online=1) if s["type"] == "node" else s for s in CLUSTER_STATUS]
api, _ = _migration_api(preflight=PREFLIGHT_BLOCKED, status=all_online)
result = _driver(api).get_vm_migration_preflight("100")
assert result["running"] is False
assert result["allowed_nodes"] == []
assert result["refused_nodes"] == {
"pve2": "storage local-zfs is not available there",
"pve3": "HA resource vm:101 blocks it (resource-affinity rule); "
"mapped resource gpu-a is not available there",
}
assert result["local_resources"] == ["hostpci0", "usb0"]
def test_unknown_reason_is_passed_on(self):
preflight = {
"running": 0,
"allowed_nodes": [],
"not_allowed_nodes": {"pve2": {"some-future-check": ["x", "y"]}},
"local_disks": [],
"local_resources": [],
}
api, _ = _migration_api(preflight=preflight)
refused = _driver(api).get_vm_migration_preflight("web")["refused_nodes"]
assert refused["pve2"] == "some-future-check: x, y"
def test_unknown_guest(self):
api, _ = _migration_api()
with pytest.raises(ValueError, match="nope"):
_driver(api).get_vm_migration_preflight("nope")
class TestPreflightLxc:
def test_members_and_local_volumes(self):
api, node = _migration_api()
result = _driver(api).get_vm_migration_preflight("dns")
assert result == {
"running": True,
"allowed_nodes": ["pve2"],
"refused_nodes": {"pve3": "node offline"},
# ceph-vm is shared; local-lvm is not.
"local_disks": ["local-lvm:vm-101-disk-0", "local-lvm:vm-101-disk-2"],
# A bind mount that is not marked shared stops the migration.
"local_resources": ["mp1"],
}
node.qemu.return_value.migrate.get.assert_not_called()
def test_stopped_container(self):
api, node = _migration_api()
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
assert _driver(api).get_vm_migration_preflight("101")["running"] is False
class TestMigrateQemu:
def test_running_vm_moves_online_with_its_local_disks(self):
api, node = _migration_api()
_driver(api).migrate_vm("web", "pve2")
node.qemu.return_value.migrate.post.assert_called_once_with(
target="pve2", online=1, **{"with-local-disks": 1}
)
node.tasks.assert_called_with(UPID_QEMU)
def test_target_storage_and_offline(self):
preflight = dict(PREFLIGHT_RUNNING, local_disks=[])
api, node = _migration_api(preflight=preflight)
_driver(api).migrate_vm("web", "pve2", online=False, target_storage="ceph-vm")
node.qemu.return_value.migrate.post.assert_called_once_with(
target="pve2", targetstorage="ceph-vm"
)
def test_stopped_vm_is_not_moved_online(self):
preflight = dict(PREFLIGHT_RUNNING, running=0, local_disks=[])
api, node = _migration_api(preflight=preflight)
_driver(api).migrate_vm("idle", "pve2")
node.qemu.return_value.migrate.post.assert_called_once_with(target="pve2")
def test_failure_raises_with_the_task_logs_errors(self):
api, node = _migration_api(
task={"status": "stopped", "exitstatus": "migration problems"}, log=FAILED_LOG
)
with pytest.raises(RuntimeError) as exc:
_driver(api).migrate_vm("web", "pve2")
message = str(exc.value)
assert "online migrate failure - remote command failed with exit code 255" in message
assert "migration finished with problems (duration 00:00:03)" in message
assert "starting VM 100" not in message
node.tasks.return_value.log.get.assert_called_once()
def test_failure_without_error_lines_keeps_the_task_status(self):
api, _ = _migration_api(
task={"status": "stopped", "exitstatus": "migration aborted"},
log=[{"n": 1, "t": "starting migration"}],
)
with pytest.raises(RuntimeError, match="migration aborted"):
_driver(api).migrate_vm("web", "pve2")
def test_refused_request_raises(self):
api, node = _migration_api()
node.qemu.return_value.migrate.post.side_effect = RuntimeError(
"500 Internal Server Error: VM is locked (backup)"
)
with pytest.raises(RuntimeError, match="VM is locked"):
_driver(api).migrate_vm("web", "pve2")
class TestMigrateLxc:
def test_running_container_moves_in_restart_mode(self):
api, node = _migration_api()
_driver(api).migrate_vm("dns", "pve2", target_storage="local-lvm")
node.lxc.return_value.migrate.post.assert_called_once_with(
target="pve2", restart=1, **{"target-storage": "local-lvm"}
)
node.tasks.assert_called_with(UPID_LXC)
def test_stopped_container(self):
api, node = _migration_api()
node.lxc.return_value.status.current.get.return_value = {"status": "stopped"}
_driver(api).migrate_vm("dns", "pve2")
node.lxc.return_value.migrate.post.assert_called_once_with(target="pve2")
class TestTargetRefusedBeforeTheApi:
@pytest.mark.parametrize("target", ["pve1", "pve9", "pve2; reboot"])
def test_not_another_member(self, target):
api, node = _migration_api()
with pytest.raises(ValueError, match="not another member"):
_driver(api).migrate_vm("web", target)
node.qemu.return_value.migrate.post.assert_not_called()
def _ha_status(node: str, crm_state: str) -> list[dict[str, Any]]:
return [
{"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"},
{
"id": "service:vm:100",
"type": "service",
"sid": "vm:100",
"node": node,
"crm_state": crm_state,
"state": crm_state,
"status": f"vm:100 ({node}, {crm_state})",
"request_state": "started",
},
]
class TestMigrateHaManaged:
"""PVE hands an HA guest's migration to the HA manager: the hamigrate task only
queues the request, so the driver follows the HA manager until the guest is there."""
def _api(self, states: list[list[dict[str, Any]]]) -> tuple[MagicMock, MagicMock]:
api, node = _migration_api()
node.qemu.return_value.migrate.post.return_value = UPID_HA
api.cluster.ha.status.current.get.side_effect = states
return api, node
def test_waits_until_the_guest_runs_on_the_target(self):
api, _ = self._api(
[
_ha_status("pve1", "started"),
_ha_status("pve1", "migrate"),
_ha_status("pve2", "started"),
]
)
with patch("napalm_proxmox.vm_migrate_mixin.sleep") as sleep:
_driver(api).migrate_vm("web", "pve2")
assert api.cluster.ha.status.current.get.call_count == 3
assert sleep.call_count == 2
def test_back_on_the_source_node_is_a_failure_with_the_migration_log(self):
api, node = self._api([_ha_status("pve1", "migrate"), _ha_status("pve1", "started")])
node.tasks.get.return_value = [
{
"upid": UPID_QEMU,
"type": "qmigrate",
"id": "100",
"starttime": 0x672AB002 + 12,
"status": "migration problems",
}
]
node.tasks.return_value.log.get.return_value = FAILED_LOG
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
with pytest.raises(RuntimeError, match="online migrate failure"):
_driver(api).migrate_vm("web", "pve2")
node.tasks.get.assert_called_once_with(vmid=100, typefilter="qmigrate", limit=1)
def test_error_state_raises(self):
api, node = self._api([_ha_status("pve1", "error")])
node.tasks.get.return_value = []
with patch("napalm_proxmox.vm_migrate_mixin.sleep"):
with pytest.raises(RuntimeError, match="error"):
_driver(api).migrate_vm("web", "pve2")
def test_times_out(self):
api, _ = self._api([_ha_status("pve1", "migrate")] * 50)
clock = iter(range(0, 10000, 30))
with (
patch("napalm_proxmox.vm_migrate_mixin.sleep"),
patch("napalm_proxmox.vm_migrate_mixin.monotonic", lambda: next(clock)),
):
with pytest.raises(RuntimeError, match="timed out"):
_driver(api).migrate_vm("web", "pve2", timeout=120)
class TestNodeMaintenance:
def _driver(self) -> tuple[Any, MagicMock]:
api, _ = _migration_api()
drv = _driver(api)
drv._node_ssh = MagicMock() # type: ignore[method-assign]
drv._run_node_command = MagicMock(return_value="") # type: ignore[method-assign]
return drv, drv._run_node_command
def test_enable(self):
drv, run = self._driver()
drv.set_node_maintenance("pve2", True)
run.assert_called_once_with(
"ha-manager crm-command node-maintenance enable pve2", timeout=drv.timeout
)
def test_disable(self):
drv, run = self._driver()
drv.set_node_maintenance("pve3", False)
run.assert_called_once_with(
"ha-manager crm-command node-maintenance disable pve3", timeout=drv.timeout
)
@pytest.mark.parametrize("node", ["pve9", "pve2; reboot", "$(reboot)"])
def test_not_a_member_runs_nothing(self, node):
drv, run = self._driver()
with pytest.raises(ValueError, match="not a member"):
drv.set_node_maintenance(node, True)
run.assert_not_called()
def test_no_ssh(self):
drv, run = self._driver()
drv._node_ssh.side_effect = OSError("Authentication failed")
with pytest.raises(RuntimeError, match="SSH"):
drv.set_node_maintenance("pve2", True)
run.assert_not_called()
def test_command_refused(self):
drv, run = self._driver()
run.side_effect = RuntimeError("Command failed (exit 255): ...\nno such node 'pve2'")
with pytest.raises(RuntimeError, match="no such node"):
drv.set_node_maintenance("pve2", True)