"""Moving a guest to another node: preflight, migration, HA maintenance mode. Payloads: ``GET /nodes/{node}/qemu/{vmid}/migrate`` follows qemu-server's ``migrate_vm_precondition`` (PVE 9: ``not_allowed_nodes`` may name ``unavailable_storages``, ``unavailable-resources`` and ``blocking-ha-resources``); the task log follows ``/nodes/{node}/tasks/{upid}/log`` (``n``/``t`` rows). A guest that HA manages migrates through a ``hamigrate`` task that only hands the request to the HA manager. """ from __future__ import annotations from typing import Any from unittest.mock import MagicMock, patch import pytest from tests.test_clusters import CLUSTER_RESOURCES, CLUSTER_STATUS, GIB, _api, _driver, _storage_row QEMU_LIST = [ {"vmid": 100, "name": "web", "status": "running", "maxmem": 4 * GIB}, {"vmid": 102, "name": "idle", "status": "stopped", "maxmem": 2 * GIB}, ] LXC_LIST = [{"vmid": "101", "name": "dns", "status": "running", "type": "lxc"}] PREFLIGHT_RUNNING = { "running": 1, "allowed_nodes": ["pve2", "pve3"], "not_allowed_nodes": {}, "local_disks": [ { "volid": "local-lvm:vm-100-disk-0", "size": 32 * GIB, "drivename": "scsi0", "is_attached": 1, "referenced_in_config": 1, }, {"volid": "local:iso/debian-13.iso", "cdrom": 1, "size": 700 * 1024**2}, # Regenerated on the target: neither a disk to copy nor a blocker. {"volid": "local-lvm:vm-100-cloudinit", "cdrom": 1, "size": 4 * 1024**2}, ], "local_resources": [], "mapped-resources": [], "mapped-resource-info": {}, "has-dbus-vmstate": 1, } PREFLIGHT_BLOCKED = { "running": 0, "allowed_nodes": [], "not_allowed_nodes": { "pve2": {"unavailable_storages": ["local-zfs"]}, "pve3": { "blocking-ha-resources": [{"sid": "vm:101", "cause": "resource-affinity"}], "unavailable-resources": ["gpu-a"], }, }, "local_disks": [], "local_resources": ["hostpci0", "usb0"], "mapped-resources": ["gpu-a"], "mapped-resource-info": {"gpu-a": {"live-migration": 0}}, "has-dbus-vmstate": 1, } LXC_CONFIG = { "hostname": "dns", "rootfs": "local-lvm:vm-101-disk-0,size=8G", "mp0": "ceph-vm:vm-101-disk-1,mp=/data,size=32G", "mp1": "/srv/share,mp=/share", "mp2": "/mnt/nfs,mp=/nfs,shared=1", "unused0": "local-lvm:vm-101-disk-2", "net0": "name=eth0,bridge=vmbr0,ip=dhcp", "digest": "abc", } UPID_QEMU = "UPID:pve1:000ABCDE:0123ABCD:672AB000:qmigrate:100:root@pam:" UPID_LXC = "UPID:pve1:000ABCDF:0123ABCE:672AB001:vzmigrate:101:root@pam:" UPID_HA = "UPID:pve1:000ABCE0:0123ABCF:672AB002:hamigrate:100:root@pam:" FAILED_LOG = [ {"n": 1, "t": "2026-11-01 10:00:00 starting migration of VM 100 to node 'pve2' (10.0.0.2)"}, {"n": 2, "t": "2026-11-01 10:00:01 starting VM 100 on remote node 'pve2'"}, { "n": 3, "t": "2026-11-01 10:00:02 ERROR: online migrate failure - remote command failed " "with exit code 255", }, {"n": 4, "t": "2026-11-01 10:00:02 aborting phase 2 - cleanup resources"}, { "n": 5, "t": "2026-11-01 10:00:03 ERROR: migration finished with problems (duration 00:00:03)", }, {"n": 6, "t": "TASK ERROR: migration problems"}, ] def _migration_api( preflight: dict[str, Any] | None = None, task: dict[str, Any] | None = None, log: list[dict[str, Any]] | None = None, status: list[dict[str, Any]] | None = None, ) -> tuple[MagicMock, MagicMock]: resources = [ *CLUSTER_RESOURCES, _storage_row("pve1", "local", "dir", 0, "iso,vztmpl,backup", 100 * GIB, 20 * GIB), ] api = _api(cluster_status=status, resources=resources) node = api.nodes("pve1") node.qemu.get.return_value = QEMU_LIST node.lxc.get.return_value = LXC_LIST node.qemu.return_value.migrate.get.return_value = preflight or PREFLIGHT_RUNNING node.qemu.return_value.migrate.post.return_value = UPID_QEMU node.lxc.return_value.migrate.post.return_value = UPID_LXC node.lxc.return_value.status.current.get.return_value = {"status": "running", "vmid": 101} node.lxc.return_value.config.get.return_value = LXC_CONFIG node.tasks.return_value.status.get.return_value = task or { "status": "stopped", "exitstatus": "OK", } node.tasks.return_value.log.get.return_value = log or [] return api, node class TestPreflightQemu: def test_allowed_nodes_and_local_disks(self): api, node = _migration_api() result = _driver(api).get_vm_migration_preflight("web") assert result == { "running": True, # pve3 is offline: the platform would allow it, it cannot take the VM now. "allowed_nodes": ["pve2"], "refused_nodes": {"pve3": "node offline"}, "local_disks": ["local-lvm:vm-100-disk-0"], # A local ISO in a CD drive stops the migration. "local_resources": ["cdrom=local:iso/debian-13.iso"], } node.qemu.assert_any_call(100) def test_refusal_reasons_in_plain_words(self): all_online = [dict(s, online=1) if s["type"] == "node" else s for s in CLUSTER_STATUS] api, _ = _migration_api(preflight=PREFLIGHT_BLOCKED, status=all_online) result = _driver(api).get_vm_migration_preflight("100") assert result["running"] is False assert result["allowed_nodes"] == [] assert result["refused_nodes"] == { "pve2": "storage local-zfs is not available there", "pve3": "HA resource vm:101 blocks it (resource-affinity rule); " "mapped resource gpu-a is not available there", } assert result["local_resources"] == ["hostpci0", "usb0"] def test_unknown_reason_is_passed_on(self): preflight = { "running": 0, "allowed_nodes": [], "not_allowed_nodes": {"pve2": {"some-future-check": ["x", "y"]}}, "local_disks": [], "local_resources": [], } api, _ = _migration_api(preflight=preflight) refused = _driver(api).get_vm_migration_preflight("web")["refused_nodes"] assert refused["pve2"] == "some-future-check: x, y" def test_unknown_guest(self): api, _ = _migration_api() with pytest.raises(ValueError, match="nope"): _driver(api).get_vm_migration_preflight("nope") class TestPreflightLxc: def test_members_and_local_volumes(self): api, node = _migration_api() result = _driver(api).get_vm_migration_preflight("dns") assert result == { "running": True, "allowed_nodes": ["pve2"], "refused_nodes": {"pve3": "node offline"}, # ceph-vm is shared; local-lvm is not. "local_disks": ["local-lvm:vm-101-disk-0", "local-lvm:vm-101-disk-2"], # A bind mount that is not marked shared stops the migration. "local_resources": ["mp1"], } node.qemu.return_value.migrate.get.assert_not_called() def test_stopped_container(self): api, node = _migration_api() node.lxc.return_value.status.current.get.return_value = {"status": "stopped"} assert _driver(api).get_vm_migration_preflight("101")["running"] is False class TestMigrateQemu: def test_running_vm_moves_online_with_its_local_disks(self): api, node = _migration_api() _driver(api).migrate_vm("web", "pve2") node.qemu.return_value.migrate.post.assert_called_once_with( target="pve2", online=1, **{"with-local-disks": 1} ) node.tasks.assert_called_with(UPID_QEMU) def test_target_storage_and_offline(self): preflight = dict(PREFLIGHT_RUNNING, local_disks=[]) api, node = _migration_api(preflight=preflight) _driver(api).migrate_vm("web", "pve2", online=False, target_storage="ceph-vm") node.qemu.return_value.migrate.post.assert_called_once_with( target="pve2", targetstorage="ceph-vm" ) def test_stopped_vm_is_not_moved_online(self): preflight = dict(PREFLIGHT_RUNNING, running=0, local_disks=[]) api, node = _migration_api(preflight=preflight) _driver(api).migrate_vm("idle", "pve2") node.qemu.return_value.migrate.post.assert_called_once_with(target="pve2") def test_failure_raises_with_the_task_logs_errors(self): api, node = _migration_api( task={"status": "stopped", "exitstatus": "migration problems"}, log=FAILED_LOG ) with pytest.raises(RuntimeError) as exc: _driver(api).migrate_vm("web", "pve2") message = str(exc.value) assert "online migrate failure - remote command failed with exit code 255" in message assert "migration finished with problems (duration 00:00:03)" in message assert "starting VM 100" not in message node.tasks.return_value.log.get.assert_called_once() def test_failure_without_error_lines_keeps_the_task_status(self): api, _ = _migration_api( task={"status": "stopped", "exitstatus": "migration aborted"}, log=[{"n": 1, "t": "starting migration"}], ) with pytest.raises(RuntimeError, match="migration aborted"): _driver(api).migrate_vm("web", "pve2") def test_refused_request_raises(self): api, node = _migration_api() node.qemu.return_value.migrate.post.side_effect = RuntimeError( "500 Internal Server Error: VM is locked (backup)" ) with pytest.raises(RuntimeError, match="VM is locked"): _driver(api).migrate_vm("web", "pve2") class TestMigrateLxc: def test_running_container_moves_in_restart_mode(self): api, node = _migration_api() _driver(api).migrate_vm("dns", "pve2", target_storage="local-lvm") node.lxc.return_value.migrate.post.assert_called_once_with( target="pve2", restart=1, **{"target-storage": "local-lvm"} ) node.tasks.assert_called_with(UPID_LXC) def test_stopped_container(self): api, node = _migration_api() node.lxc.return_value.status.current.get.return_value = {"status": "stopped"} _driver(api).migrate_vm("dns", "pve2") node.lxc.return_value.migrate.post.assert_called_once_with(target="pve2") class TestTargetRefusedBeforeTheApi: @pytest.mark.parametrize("target", ["pve1", "pve9", "pve2; reboot"]) def test_not_another_member(self, target): api, node = _migration_api() with pytest.raises(ValueError, match="not another member"): _driver(api).migrate_vm("web", target) node.qemu.return_value.migrate.post.assert_not_called() def _ha_status(node: str, crm_state: str) -> list[dict[str, Any]]: return [ {"id": "quorum", "type": "quorum", "node": "pve1", "quorate": 1, "status": "OK"}, { "id": "service:vm:100", "type": "service", "sid": "vm:100", "node": node, "crm_state": crm_state, "state": crm_state, "status": f"vm:100 ({node}, {crm_state})", "request_state": "started", }, ] class TestMigrateHaManaged: """PVE hands an HA guest's migration to the HA manager: the hamigrate task only queues the request, so the driver follows the HA manager until the guest is there.""" def _api(self, states: list[list[dict[str, Any]]]) -> tuple[MagicMock, MagicMock]: api, node = _migration_api() node.qemu.return_value.migrate.post.return_value = UPID_HA api.cluster.ha.status.current.get.side_effect = states return api, node def test_waits_until_the_guest_runs_on_the_target(self): api, _ = self._api( [ _ha_status("pve1", "started"), _ha_status("pve1", "migrate"), _ha_status("pve2", "started"), ] ) with patch("napalm_proxmox.vm_migrate_mixin.sleep") as sleep: _driver(api).migrate_vm("web", "pve2") assert api.cluster.ha.status.current.get.call_count == 3 assert sleep.call_count == 2 def test_back_on_the_source_node_is_a_failure_with_the_migration_log(self): api, node = self._api([_ha_status("pve1", "migrate"), _ha_status("pve1", "started")]) node.tasks.get.return_value = [ { "upid": UPID_QEMU, "type": "qmigrate", "id": "100", "starttime": 0x672AB002 + 12, "status": "migration problems", } ] node.tasks.return_value.log.get.return_value = FAILED_LOG with patch("napalm_proxmox.vm_migrate_mixin.sleep"): with pytest.raises(RuntimeError, match="online migrate failure"): _driver(api).migrate_vm("web", "pve2") node.tasks.get.assert_called_once_with(vmid=100, typefilter="qmigrate", limit=1) def test_error_state_raises(self): api, node = self._api([_ha_status("pve1", "error")]) node.tasks.get.return_value = [] with patch("napalm_proxmox.vm_migrate_mixin.sleep"): with pytest.raises(RuntimeError, match="error"): _driver(api).migrate_vm("web", "pve2") def test_times_out(self): api, _ = self._api([_ha_status("pve1", "migrate")] * 50) clock = iter(range(0, 10000, 30)) with ( patch("napalm_proxmox.vm_migrate_mixin.sleep"), patch("napalm_proxmox.vm_migrate_mixin.monotonic", lambda: next(clock)), ): with pytest.raises(RuntimeError, match="timed out"): _driver(api).migrate_vm("web", "pve2", timeout=120) class TestNodeMaintenance: def _driver(self) -> tuple[Any, MagicMock]: api, _ = _migration_api() drv = _driver(api) drv._node_ssh = MagicMock() # type: ignore[method-assign] drv._run_node_command = MagicMock(return_value="") # type: ignore[method-assign] return drv, drv._run_node_command def test_enable(self): drv, run = self._driver() drv.set_node_maintenance("pve2", True) run.assert_called_once_with( "ha-manager crm-command node-maintenance enable pve2", timeout=drv.timeout ) def test_disable(self): drv, run = self._driver() drv.set_node_maintenance("pve3", False) run.assert_called_once_with( "ha-manager crm-command node-maintenance disable pve3", timeout=drv.timeout ) @pytest.mark.parametrize("node", ["pve9", "pve2; reboot", "$(reboot)"]) def test_not_a_member_runs_nothing(self, node): drv, run = self._driver() with pytest.raises(ValueError, match="not a member"): drv.set_node_maintenance(node, True) run.assert_not_called() def test_no_ssh(self): drv, run = self._driver() drv._node_ssh.side_effect = OSError("Authentication failed") with pytest.raises(RuntimeError, match="SSH"): drv.set_node_maintenance("pve2", True) run.assert_not_called() def test_command_refused(self): drv, run = self._driver() run.side_effect = RuntimeError("Command failed (exit 255): ...\nno such node 'pve2'") with pytest.raises(RuntimeError, match="no such node"): drv.set_node_maintenance("pve2", True)