feat(running-backups): what a restart of a PVE node, a guest or a PBS would abort
driver / lint (pull_request) Successful in 13s
driver / standard (pull_request) Successful in 14s
driver / types (pull_request) Successful in 54s
driver / test (3.13) (pull_request) Successful in 1m2s
driver / test (3.12) (pull_request) Successful in 1m5s
driver / lowest (pull_request) Successful in 31s
driver / build (pull_request) Successful in 26s
driver / security (pull_request) Successful in 54s
CI / driver (pull_request) Successful in 2m10s
driver / publish (pull_request) Skipped

netOrk holds a host's restart after patching while a backup runs on it, and
a guest's restart while its hypervisor is backing it up (NetOrk/netork#2318).
It asks right before the restart, so both drivers answer
get_running_backups() from napalm-device-types 4.34.0:

- PVE: the node's active vzdump tasks, each with the guests it is taking now
  (its own guest, or for a job the guests locked for their backup), and a
  guest locked without a task on its own, since the task list can lag and a
  lock can outlive its task.
- PBS: the running backup, sync, verify, gc, prune and tape tasks; a restore
  or housekeeping is no backup.

A failed read raises, so netOrk reads it as "not checked", never as "none".
This commit is contained in:
Christian Manivong committed 2026-11-10 23:45:48 +01:00
1 parent b1252878a5
commit 3877a5425a
6 files changed
+289 -2

No files matched your search

+7 -1
View File
@@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- What a restart would abort, on both drivers (NetOrk/netork#2318), through
napalm-device-types' `RunningBackupsMixin` (4.34.0): `get_running_backups()` lists on
Proxmox VE the node's active vzdump tasks with the guests each is taking now -- the task's
own guest, or for a job the guests locked for their backup (`lock: backup`) -- and a guest
locked without a task on its own; on a backup server the running backup, sync, verify,
garbage collection, prune and tape tasks. A failed read raises.
- Log sources on both drivers (NetOrk/netork#2049), through napalm-device-types'
`LogSourcesMixin` (4.30.0): `get_log_sources()` reads the units journald holds entries for
and the log files running processes hold open, with the service or container writing each,
@@ -27,7 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
in its own daemons; `get_clusters()` and `get_backup_jobs()` keep them.
### Changed
- Requires napalm-device-types 4.26.0.
- Requires napalm-device-types 4.34.0.
## [0.10.0] - 2026-11-03
+2
View File
@@ -41,6 +41,7 @@ from napalm_proxmox._base import ProxmoxBase
from napalm_proxmox.host_mixin import ProxmoxHostMixin
from napalm_proxmox.pbs_backups import ProxmoxBackupServerMixin
from napalm_proxmox.pbs_host import ProxmoxPbsHostMixin
from napalm_proxmox.running_backups import PbsRunningBackupsMixin
from napalm_proxmox.scheduled_jobs import PBS_PLATFORM_SOURCES, ProxmoxScheduledJobsMixin
logger = logging.getLogger(__name__)
@@ -52,6 +53,7 @@ PBS_NODE = "localhost"
class ProxmoxBackupDriver(
ProxmoxBase,
ProxmoxBackupServerMixin,
PbsRunningBackupsMixin,
ProxmoxPbsHostMixin,
ProxmoxHostMixin,
BackupServerDriver,
+2
View File
@@ -55,6 +55,7 @@ from napalm_proxmox.host_mixin import ProxmoxHostMixin
from napalm_proxmox.interfaces_mixin import ProxmoxInterfaceMixin
from napalm_proxmox.lldp_mixin import ProxmoxLLDPMixin
from napalm_proxmox.routing_mixin import ProxmoxRoutingMixin
from napalm_proxmox.running_backups import ProxmoxRunningBackupsMixin
from napalm_proxmox.scheduled_jobs import PVE_PLATFORM_SOURCES, ProxmoxScheduledJobsMixin
from napalm_proxmox.sdn_mixin import ProxmoxSDNMixin
from napalm_proxmox.system_mixin import ProxmoxSystemMixin
@@ -98,6 +99,7 @@ class ProxmoxDriver(
SystemIdentityMixin,
SystemdServicesMixin,
ProxmoxScheduledJobsMixin,
ProxmoxRunningBackupsMixin,
HostStatusMixin,
LicenseMixin,
SnmpAgentMixin,
+126
View File
@@ -0,0 +1,126 @@
"""get_running_backups() for Proxmox VE and Proxmox Backup Server.
What a restart would abort (napalm-device-types' ``RunningBackupsMixin``):
- **PVE:** the node's active vzdump tasks (``GET /nodes/<node>/tasks?source=active&
typefilter=vzdump``) and the guests on the node locked for their backup
(``GET /cluster/resources?type=vm``, ``lock: backup``). A task started for one
guest names it in its id; a job's task does not, and the guests it is taking now
are the locked ones. A guest locked without a task -- the list can lag behind,
a lock can outlive its task -- is listed on its own: its restart waits too.
- **PBS:** the running tasks that are a backup or one of the server's jobs
(``GET /nodes/localhost/tasks?running=1``). A restore or housekeeping is not.
PVE does not tell which scheduled job started a vzdump task, so ``job`` is None
there. A failed read raises: an empty list would say nothing runs.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from napalm_device_types import RunningBackupDict, RunningBackupKind, RunningBackupsMixin
_JsonDict = dict[str, Any]
#: PBS worker types that are a backup or one of a backup server's jobs.
_PBS_KINDS: dict[str, RunningBackupKind] = {
"backup": "backup",
"sync": "sync",
"syncjob": "sync",
"verify": "verify",
"verificationjob": "verify",
"verify_group": "verify",
"verify_snapshot": "verify",
"garbage_collection": "gc",
"prune": "prune",
"prunejob": "prune",
"tape-backup": "tape",
"tape-backup-job": "tape",
}
def _since(task: _JsonDict) -> int | None:
value = task.get("starttime")
return int(value) if isinstance(value, int | float) and value else None
def _text(value: Any) -> str | None:
return str(value) if value not in (None, "") else None
class ProxmoxRunningBackupsMixin(RunningBackupsMixin):
"""PVE: the node's vzdump tasks and the guests locked for their backup."""
_api: Any
_node_name: str
if TYPE_CHECKING: # pragma: no cover - the base's, declared for type checkers
def _node_api(self) -> Any: ...
def _locked_guests(self) -> list[str]:
"""The guests on this node locked for their backup, in the API's order."""
return [
str(guest["vmid"])
for guest in self._api.cluster.resources.get(type="vm") or []
if guest.get("node") == self._node_name and guest.get("lock") == "backup"
]
def _vzdump_tasks(self) -> list[_JsonDict]:
tasks = self._node_api().tasks.get(source="active", typefilter="vzdump") or []
return [t for t in tasks if t.get("type") == "vzdump"]
def get_running_backups(self) -> list[RunningBackupDict]:
tasks = self._vzdump_tasks()
named = {str(t["id"]) for t in tasks if t.get("id")}
unnamed = [g for g in self._locked_guests() if g not in named]
found = [_vzdump(task, unnamed) for task in tasks]
if all(t.get("id") for t in tasks):
found += [_locked(guest) for guest in unnamed]
return found
def _vzdump(task: _JsonDict, unnamed: list[str]) -> RunningBackupDict:
"""A vzdump task: its own guest, or -- a job's task -- the guests locked now."""
guest = _text(task.get("id"))
return {
"kind": "backup",
"job": None,
"task": _text(task.get("upid")),
"guests": [guest] if guest else list(unnamed),
"since": _since(task),
}
def _locked(guest: str) -> RunningBackupDict:
"""A guest locked for its backup without a task that takes it."""
return {"kind": "backup", "job": None, "task": None, "guests": [guest], "since": None}
class PbsRunningBackupsMixin(RunningBackupsMixin):
"""PBS: the running backup, sync, verify, gc, prune and tape tasks."""
_node_name: str
if TYPE_CHECKING: # pragma: no cover - the backup-server mixin's, for type checkers
def _pbs(self, method: str, path: str, **params: Any) -> Any: ...
def get_running_backups(self) -> list[RunningBackupDict]:
tasks = self._pbs("get", f"nodes/{self._node_name}/tasks", running=1) or []
found: list[RunningBackupDict] = []
for task in tasks:
kind = _PBS_KINDS.get(str(task.get("worker_type") or ""))
if kind is None:
continue
found.append(
{
"kind": kind,
"job": _text(task.get("worker_id")),
"task": _text(task.get("upid")),
"guests": [],
"since": _since(task),
}
)
return found
+1 -1
View File
@@ -21,7 +21,7 @@ classifiers = [
requires-python = ">=3.12"
dependencies = [
"napalm>=5.0.0",
"napalm_device_types>=4.30.0", # LogSourcesMixin
"napalm_device_types>=4.34.0", # RunningBackupsMixin
"paramiko>=5.0.0", # CVE-2026-44405; imported directly for the node's SSH session (_base.py)
"proxmoxer>=2.0.0",
"netaddr>=0.9.0",
+151
View File
@@ -0,0 +1,151 @@
"""get_running_backups: what a restart of the node, or of one of its guests, would abort.
PVE: the node's active vzdump tasks, and the guests locked for their backup
(``lock: backup``); a task names its guest when it backs up one, else the
guests it is taking now are the locked ones. PBS: the running backup, sync,
verify, garbage collection, prune and tape tasks. A failed read raises: an
empty list would say nothing runs (NetOrk/netork#2318).
"""
from __future__ import annotations
import pytest
from proxmoxer.core import ResourceException
from napalm_proxmox import ProxmoxBackupDriver
from tests.pbs_fake import FakePbsApi
T0 = 1793660400
UPID_JOB = "UPID:pve1:0001F2A3:00C4B1D2:6731A0C0:vzdump::root@pam:"
UPID_ONE = "UPID:pve1:0001F2A4:00C4B1D3:6731A0D0:vzdump:105:root@pam:"
def _pve(driver, tasks, resources):
node = driver._api.nodes.return_value
node.tasks.get.return_value = tasks
driver._api.cluster.resources.get.return_value = resources
return node
def test_nothing_runs(driver):
_pve(driver, [], [{"vmid": 101, "node": "pve1", "type": "qemu"}])
assert driver.get_running_backups() == []
def test_asks_for_the_nodes_active_vzdump_tasks_and_the_guests(driver):
node = _pve(driver, [], [])
driver.get_running_backups()
node.tasks.get.assert_called_once_with(source="active", typefilter="vzdump")
driver._api.cluster.resources.get.assert_called_once_with(type="vm")
def test_a_job_takes_the_guests_locked_on_this_node(driver):
_pve(
driver,
[{"upid": UPID_JOB, "type": "vzdump", "id": "", "starttime": T0}],
[
{"vmid": 101, "node": "pve1", "type": "qemu", "lock": "backup"},
{"vmid": 102, "node": "pve1", "type": "lxc"},
{"vmid": 103, "node": "pve2", "type": "qemu", "lock": "backup"},
{"vmid": 104, "node": "pve1", "type": "qemu", "lock": "migrate"},
],
)
assert driver.get_running_backups() == [
{"kind": "backup", "job": None, "task": UPID_JOB, "guests": ["101"], "since": T0}
]
def test_a_task_for_one_guest_names_it(driver):
_pve(
driver,
[{"upid": UPID_ONE, "type": "vzdump", "id": "105", "starttime": T0}],
[{"vmid": 105, "node": "pve1", "type": "qemu", "lock": "backup"}],
)
assert driver.get_running_backups() == [
{"kind": "backup", "job": None, "task": UPID_ONE, "guests": ["105"], "since": T0}
]
def test_a_guest_locked_without_a_task_still_counts(driver):
"""The task list can lag, or a lock outlive its task: the guest is not restarted."""
_pve(driver, [], [{"vmid": 101, "node": "pve1", "type": "qemu", "lock": "backup"}])
assert driver.get_running_backups() == [
{"kind": "backup", "job": None, "task": None, "guests": ["101"], "since": None}
]
def test_another_tasks_type_is_left_out(driver):
_pve(driver, [{"upid": "UPID:pve1:1:2:3:qmstart:101:root@pam:", "type": "qmstart"}], [])
assert driver.get_running_backups() == []
def test_a_failed_read_raises(driver):
driver._api.nodes.return_value.tasks.get.side_effect = ResourceException(
403, "Forbidden", "Permission check failed (/nodes/pve1, Sys.Audit)"
)
with pytest.raises(ResourceException):
driver.get_running_backups()
# --------------------------------------------------------------------------- #
# Proxmox Backup Server
# --------------------------------------------------------------------------- #
def _pbs(tasks):
drv = ProxmoxBackupDriver("pbs1", "root", "secret")
drv._api = FakePbsApi(**{"nodes/localhost/tasks?running=1": tasks})
drv._node_name = "localhost"
return drv
def _task(worker_type: str, worker_id: str | None, n: int = 1) -> dict:
upid = f"UPID:localhost:{n:08X}:00000000:00000000:6731A0C0:{worker_type}:{worker_id or ''}:root@pam:"
return {"upid": upid, "worker_type": worker_type, "worker_id": worker_id, "starttime": T0 + n}
def test_pbs_kinds():
tasks = [
_task("backup", "store1:vm/101", 1),
_task("syncjob", "store1:offsite:store1::s-1a2b", 2),
_task("verificationjob", "store1:v-3c4d", 3),
_task("verify_snapshot", "store1:vm/101/2026-11-01T00:00:00Z", 4),
_task("garbage_collection", "store1", 5),
_task("prunejob", "store1:p-5e6f", 6),
_task("tape-backup-job", "weekly", 7),
]
found = _pbs(tasks).get_running_backups()
assert [(b["kind"], b["job"]) for b in found] == [
("backup", "store1:vm/101"),
("sync", "store1:offsite:store1::s-1a2b"),
("verify", "store1:v-3c4d"),
("verify", "store1:vm/101/2026-11-01T00:00:00Z"),
("gc", "store1"),
("prune", "store1:p-5e6f"),
("tape", "weekly"),
]
assert all(b["guests"] == [] for b in found)
assert found[0]["task"] == tasks[0]["upid"]
assert found[0]["since"] == T0 + 1
def test_pbs_leaves_out_what_is_no_backup():
"""A restore reads, a log rotation is housekeeping: neither is broken by a restart
the way a backup is."""
assert (
_pbs([_task("reader", "store1:vm/101"), _task("logrotate", None)]).get_running_backups()
== []
)
def test_pbs_nothing_runs():
assert _pbs([]).get_running_backups() == []
def test_pbs_a_failed_read_raises():
drv = _pbs([])
drv._api.responses["nodes/localhost/tasks?running=1"] = ResourceException(
403, "Forbidden", "permission check failed"
)
with pytest.raises(ResourceException):
drv.get_running_backups()