From 3877a5425ad274f3c7e440f1dc0f2a085bbe2fc8 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 10 Nov 2026 23:45:48 +0100 Subject: [PATCH] feat(running-backups): what a restart of a PVE node, a guest or a PBS would abort netOrk holds a host's restart after patching while a backup runs on it, and a guest's restart while its hypervisor is backing it up (NetOrk/netork#2318). It asks right before the restart, so both drivers answer get_running_backups() from napalm-device-types 4.34.0: - PVE: the node's active vzdump tasks, each with the guests it is taking now (its own guest, or for a job the guests locked for their backup), and a guest locked without a task on its own, since the task list can lag and a lock can outlive its task. - PBS: the running backup, sync, verify, gc, prune and tape tasks; a restore or housekeeping is no backup. A failed read raises, so netOrk reads it as "not checked", never as "none". --- CHANGELOG.md | 8 +- napalm_proxmox/backup_driver.py | 2 + napalm_proxmox/driver.py | 2 + napalm_proxmox/running_backups.py | 126 +++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_running_backups.py | 151 ++++++++++++++++++++++++++++++ 6 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 napalm_proxmox/running_backups.py create mode 100644 tests/test_running_backups.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 35860b7..5b8510f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- What a restart would abort, on both drivers (NetOrk/netork#2318), through + napalm-device-types' `RunningBackupsMixin` (4.34.0): `get_running_backups()` lists on + Proxmox VE the node's active vzdump tasks with the guests each is taking now -- the task's + own guest, or for a job the guests locked for their backup (`lock: backup`) -- and a guest + locked without a task on its own; on a backup server the running backup, sync, verify, + garbage collection, prune and tape tasks. A failed read raises. - Log sources on both drivers (NetOrk/netork#2049), through napalm-device-types' `LogSourcesMixin` (4.30.0): `get_log_sources()` reads the units journald holds entries for and the log files running processes hold open, with the service or container writing each, @@ -27,7 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 in its own daemons; `get_clusters()` and `get_backup_jobs()` keep them. ### Changed -- Requires napalm-device-types 4.26.0. +- Requires napalm-device-types 4.34.0. ## [0.10.0] - 2026-11-03 diff --git a/napalm_proxmox/backup_driver.py b/napalm_proxmox/backup_driver.py index 6ec1134..5992ddc 100644 --- a/napalm_proxmox/backup_driver.py +++ b/napalm_proxmox/backup_driver.py @@ -41,6 +41,7 @@ from napalm_proxmox._base import ProxmoxBase from napalm_proxmox.host_mixin import ProxmoxHostMixin from napalm_proxmox.pbs_backups import ProxmoxBackupServerMixin from napalm_proxmox.pbs_host import ProxmoxPbsHostMixin +from napalm_proxmox.running_backups import PbsRunningBackupsMixin from napalm_proxmox.scheduled_jobs import PBS_PLATFORM_SOURCES, ProxmoxScheduledJobsMixin logger = logging.getLogger(__name__) @@ -52,6 +53,7 @@ PBS_NODE = "localhost" class ProxmoxBackupDriver( ProxmoxBase, ProxmoxBackupServerMixin, + PbsRunningBackupsMixin, ProxmoxPbsHostMixin, ProxmoxHostMixin, BackupServerDriver, diff --git a/napalm_proxmox/driver.py b/napalm_proxmox/driver.py index 8eccd6f..8cfb54a 100644 --- a/napalm_proxmox/driver.py +++ b/napalm_proxmox/driver.py @@ -55,6 +55,7 @@ from napalm_proxmox.host_mixin import ProxmoxHostMixin from napalm_proxmox.interfaces_mixin import ProxmoxInterfaceMixin from napalm_proxmox.lldp_mixin import ProxmoxLLDPMixin from napalm_proxmox.routing_mixin import ProxmoxRoutingMixin +from napalm_proxmox.running_backups import ProxmoxRunningBackupsMixin from napalm_proxmox.scheduled_jobs import PVE_PLATFORM_SOURCES, ProxmoxScheduledJobsMixin from napalm_proxmox.sdn_mixin import ProxmoxSDNMixin from napalm_proxmox.system_mixin import ProxmoxSystemMixin @@ -98,6 +99,7 @@ class ProxmoxDriver( SystemIdentityMixin, SystemdServicesMixin, ProxmoxScheduledJobsMixin, + ProxmoxRunningBackupsMixin, HostStatusMixin, LicenseMixin, SnmpAgentMixin, diff --git a/napalm_proxmox/running_backups.py b/napalm_proxmox/running_backups.py new file mode 100644 index 0000000..993eb13 --- /dev/null +++ b/napalm_proxmox/running_backups.py @@ -0,0 +1,126 @@ +"""get_running_backups() for Proxmox VE and Proxmox Backup Server. + +What a restart would abort (napalm-device-types' ``RunningBackupsMixin``): + +- **PVE:** the node's active vzdump tasks (``GET /nodes//tasks?source=active& + typefilter=vzdump``) and the guests on the node locked for their backup + (``GET /cluster/resources?type=vm``, ``lock: backup``). A task started for one + guest names it in its id; a job's task does not, and the guests it is taking now + are the locked ones. A guest locked without a task -- the list can lag behind, + a lock can outlive its task -- is listed on its own: its restart waits too. +- **PBS:** the running tasks that are a backup or one of the server's jobs + (``GET /nodes/localhost/tasks?running=1``). A restore or housekeeping is not. + +PVE does not tell which scheduled job started a vzdump task, so ``job`` is None +there. A failed read raises: an empty list would say nothing runs. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from napalm_device_types import RunningBackupDict, RunningBackupKind, RunningBackupsMixin + +_JsonDict = dict[str, Any] + +#: PBS worker types that are a backup or one of a backup server's jobs. +_PBS_KINDS: dict[str, RunningBackupKind] = { + "backup": "backup", + "sync": "sync", + "syncjob": "sync", + "verify": "verify", + "verificationjob": "verify", + "verify_group": "verify", + "verify_snapshot": "verify", + "garbage_collection": "gc", + "prune": "prune", + "prunejob": "prune", + "tape-backup": "tape", + "tape-backup-job": "tape", +} + + +def _since(task: _JsonDict) -> int | None: + value = task.get("starttime") + return int(value) if isinstance(value, int | float) and value else None + + +def _text(value: Any) -> str | None: + return str(value) if value not in (None, "") else None + + +class ProxmoxRunningBackupsMixin(RunningBackupsMixin): + """PVE: the node's vzdump tasks and the guests locked for their backup.""" + + _api: Any + _node_name: str + + if TYPE_CHECKING: # pragma: no cover - the base's, declared for type checkers + + def _node_api(self) -> Any: ... + + def _locked_guests(self) -> list[str]: + """The guests on this node locked for their backup, in the API's order.""" + return [ + str(guest["vmid"]) + for guest in self._api.cluster.resources.get(type="vm") or [] + if guest.get("node") == self._node_name and guest.get("lock") == "backup" + ] + + def _vzdump_tasks(self) -> list[_JsonDict]: + tasks = self._node_api().tasks.get(source="active", typefilter="vzdump") or [] + return [t for t in tasks if t.get("type") == "vzdump"] + + def get_running_backups(self) -> list[RunningBackupDict]: + tasks = self._vzdump_tasks() + named = {str(t["id"]) for t in tasks if t.get("id")} + unnamed = [g for g in self._locked_guests() if g not in named] + found = [_vzdump(task, unnamed) for task in tasks] + if all(t.get("id") for t in tasks): + found += [_locked(guest) for guest in unnamed] + return found + + +def _vzdump(task: _JsonDict, unnamed: list[str]) -> RunningBackupDict: + """A vzdump task: its own guest, or -- a job's task -- the guests locked now.""" + guest = _text(task.get("id")) + return { + "kind": "backup", + "job": None, + "task": _text(task.get("upid")), + "guests": [guest] if guest else list(unnamed), + "since": _since(task), + } + + +def _locked(guest: str) -> RunningBackupDict: + """A guest locked for its backup without a task that takes it.""" + return {"kind": "backup", "job": None, "task": None, "guests": [guest], "since": None} + + +class PbsRunningBackupsMixin(RunningBackupsMixin): + """PBS: the running backup, sync, verify, gc, prune and tape tasks.""" + + _node_name: str + + if TYPE_CHECKING: # pragma: no cover - the backup-server mixin's, for type checkers + + def _pbs(self, method: str, path: str, **params: Any) -> Any: ... + + def get_running_backups(self) -> list[RunningBackupDict]: + tasks = self._pbs("get", f"nodes/{self._node_name}/tasks", running=1) or [] + found: list[RunningBackupDict] = [] + for task in tasks: + kind = _PBS_KINDS.get(str(task.get("worker_type") or "")) + if kind is None: + continue + found.append( + { + "kind": kind, + "job": _text(task.get("worker_id")), + "task": _text(task.get("upid")), + "guests": [], + "since": _since(task), + } + ) + return found diff --git a/pyproject.toml b/pyproject.toml index 81c1398..261abe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ requires-python = ">=3.12" dependencies = [ "napalm>=5.0.0", - "napalm_device_types>=4.30.0", # LogSourcesMixin + "napalm_device_types>=4.34.0", # RunningBackupsMixin "paramiko>=5.0.0", # CVE-2026-44405; imported directly for the node's SSH session (_base.py) "proxmoxer>=2.0.0", "netaddr>=0.9.0", diff --git a/tests/test_running_backups.py b/tests/test_running_backups.py new file mode 100644 index 0000000..9e23167 --- /dev/null +++ b/tests/test_running_backups.py @@ -0,0 +1,151 @@ +"""get_running_backups: what a restart of the node, or of one of its guests, would abort. + +PVE: the node's active vzdump tasks, and the guests locked for their backup +(``lock: backup``); a task names its guest when it backs up one, else the +guests it is taking now are the locked ones. PBS: the running backup, sync, +verify, garbage collection, prune and tape tasks. A failed read raises: an +empty list would say nothing runs (NetOrk/netork#2318). +""" + +from __future__ import annotations + +import pytest +from proxmoxer.core import ResourceException + +from napalm_proxmox import ProxmoxBackupDriver +from tests.pbs_fake import FakePbsApi + +T0 = 1793660400 +UPID_JOB = "UPID:pve1:0001F2A3:00C4B1D2:6731A0C0:vzdump::root@pam:" +UPID_ONE = "UPID:pve1:0001F2A4:00C4B1D3:6731A0D0:vzdump:105:root@pam:" + + +def _pve(driver, tasks, resources): + node = driver._api.nodes.return_value + node.tasks.get.return_value = tasks + driver._api.cluster.resources.get.return_value = resources + return node + + +def test_nothing_runs(driver): + _pve(driver, [], [{"vmid": 101, "node": "pve1", "type": "qemu"}]) + assert driver.get_running_backups() == [] + + +def test_asks_for_the_nodes_active_vzdump_tasks_and_the_guests(driver): + node = _pve(driver, [], []) + driver.get_running_backups() + node.tasks.get.assert_called_once_with(source="active", typefilter="vzdump") + driver._api.cluster.resources.get.assert_called_once_with(type="vm") + + +def test_a_job_takes_the_guests_locked_on_this_node(driver): + _pve( + driver, + [{"upid": UPID_JOB, "type": "vzdump", "id": "", "starttime": T0}], + [ + {"vmid": 101, "node": "pve1", "type": "qemu", "lock": "backup"}, + {"vmid": 102, "node": "pve1", "type": "lxc"}, + {"vmid": 103, "node": "pve2", "type": "qemu", "lock": "backup"}, + {"vmid": 104, "node": "pve1", "type": "qemu", "lock": "migrate"}, + ], + ) + assert driver.get_running_backups() == [ + {"kind": "backup", "job": None, "task": UPID_JOB, "guests": ["101"], "since": T0} + ] + + +def test_a_task_for_one_guest_names_it(driver): + _pve( + driver, + [{"upid": UPID_ONE, "type": "vzdump", "id": "105", "starttime": T0}], + [{"vmid": 105, "node": "pve1", "type": "qemu", "lock": "backup"}], + ) + assert driver.get_running_backups() == [ + {"kind": "backup", "job": None, "task": UPID_ONE, "guests": ["105"], "since": T0} + ] + + +def test_a_guest_locked_without_a_task_still_counts(driver): + """The task list can lag, or a lock outlive its task: the guest is not restarted.""" + _pve(driver, [], [{"vmid": 101, "node": "pve1", "type": "qemu", "lock": "backup"}]) + assert driver.get_running_backups() == [ + {"kind": "backup", "job": None, "task": None, "guests": ["101"], "since": None} + ] + + +def test_another_tasks_type_is_left_out(driver): + _pve(driver, [{"upid": "UPID:pve1:1:2:3:qmstart:101:root@pam:", "type": "qmstart"}], []) + assert driver.get_running_backups() == [] + + +def test_a_failed_read_raises(driver): + driver._api.nodes.return_value.tasks.get.side_effect = ResourceException( + 403, "Forbidden", "Permission check failed (/nodes/pve1, Sys.Audit)" + ) + with pytest.raises(ResourceException): + driver.get_running_backups() + + +# --------------------------------------------------------------------------- # +# Proxmox Backup Server +# --------------------------------------------------------------------------- # + + +def _pbs(tasks): + drv = ProxmoxBackupDriver("pbs1", "root", "secret") + drv._api = FakePbsApi(**{"nodes/localhost/tasks?running=1": tasks}) + drv._node_name = "localhost" + return drv + + +def _task(worker_type: str, worker_id: str | None, n: int = 1) -> dict: + upid = f"UPID:localhost:{n:08X}:00000000:00000000:6731A0C0:{worker_type}:{worker_id or ''}:root@pam:" + return {"upid": upid, "worker_type": worker_type, "worker_id": worker_id, "starttime": T0 + n} + + +def test_pbs_kinds(): + tasks = [ + _task("backup", "store1:vm/101", 1), + _task("syncjob", "store1:offsite:store1::s-1a2b", 2), + _task("verificationjob", "store1:v-3c4d", 3), + _task("verify_snapshot", "store1:vm/101/2026-11-01T00:00:00Z", 4), + _task("garbage_collection", "store1", 5), + _task("prunejob", "store1:p-5e6f", 6), + _task("tape-backup-job", "weekly", 7), + ] + found = _pbs(tasks).get_running_backups() + assert [(b["kind"], b["job"]) for b in found] == [ + ("backup", "store1:vm/101"), + ("sync", "store1:offsite:store1::s-1a2b"), + ("verify", "store1:v-3c4d"), + ("verify", "store1:vm/101/2026-11-01T00:00:00Z"), + ("gc", "store1"), + ("prune", "store1:p-5e6f"), + ("tape", "weekly"), + ] + assert all(b["guests"] == [] for b in found) + assert found[0]["task"] == tasks[0]["upid"] + assert found[0]["since"] == T0 + 1 + + +def test_pbs_leaves_out_what_is_no_backup(): + """A restore reads, a log rotation is housekeeping: neither is broken by a restart + the way a backup is.""" + assert ( + _pbs([_task("reader", "store1:vm/101"), _task("logrotate", None)]).get_running_backups() + == [] + ) + + +def test_pbs_nothing_runs(): + assert _pbs([]).get_running_backups() == [] + + +def test_pbs_a_failed_read_raises(): + drv = _pbs([]) + drv._api.responses["nodes/localhost/tasks?running=1"] = ResourceException( + 403, "Forbidden", "permission check failed" + ) + with pytest.raises(ResourceException): + drv.get_running_backups()