feat(running-backups): the host's own backups a restart would abort
driver / standard (pull_request) Successful in 15s
driver / lint (pull_request) Successful in 13s
driver / types (pull_request) Successful in 59s
driver / test (3.12) (pull_request) Successful in 1m3s
driver / test (3.13) (pull_request) Successful in 54s
driver / lowest (pull_request) Successful in 28s
driver / build (pull_request) Successful in 26s
driver / security (pull_request) Successful in 54s
CI / driver (pull_request) Successful in 2m25s
driver / publish (pull_request) Skipped

netOrk holds a host's restart after patching while a backup runs on it
(NetOrk/netork#2318) and asks right before the restart. On a Linux host a
file-level backup is a process of a backup tool, started by a timer, a cron
line or by hand, so get_running_backups() (napalm-device-types 4.34.0) reads
the processes, the clock and each process's cgroup in one round trip as the
login user. What a tool does comes from its subcommand: restic backup is a
backup, restic forget a prune, restic mount none; borg serve counts, as the
receiving end of another host's backup. The systemd service a backup runs
in is its job, the timer's service netOrk shows as its scheduled job.

A ps without etimes (busybox) fails the reading rather than report nothing.
This commit is contained in:
Christian Manivong committed 2026-11-10 23:45:49 +01:00
1 parent ea82d43bd4
commit 78374cfb42
5 files changed
+348 -7

No files matched your search

+10 -5
View File
@@ -1,9 +1,11 @@
"""Readings that need root to see every process: listening sockets and log sources.
"""Readings of the host's processes: listening sockets, log sources, running backups.
Both commands are napalm-device-types' and read-only. Root only widens what they
see -- ``ss`` names every process behind a socket, ``/proc/<pid>/fd`` shows every
The first two commands are napalm-device-types' and read-only. Root only widens
what they see -- ``ss`` names every process behind a socket, ``/proc/<pid>/fd`` shows every
process's open files -- so both go through sudo where the driver can, and as the
login user where it cannot; the mixins then say the reading is not ``attributed``.
The running backups need no root: every process's command line and cgroup are
readable to the login user (``_running_backups``).
"""
from __future__ import annotations
@@ -12,12 +14,15 @@ from typing import TYPE_CHECKING
from napalm_device_types import ListeningSocketsMixin, LogSourcesMixin
from napalm_linux._running_backups import LinuxRunningBackupsMixin
#: Seconds either reading may take; a busy host lists thousands of descriptors.
_READ_TIMEOUT = 60
class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin):
"""The transports for ``get_listening_sockets`` and ``get_log_sources``."""
class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin, LinuxRunningBackupsMixin):
"""The transports for ``get_listening_sockets`` and ``get_log_sources``, and
``get_running_backups``."""
if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers
+188
View File
@@ -0,0 +1,188 @@
"""The host's own backups a restart would abort (napalm-device-types' ``RunningBackupsMixin``).
A file-level backup on a Linux host is a process of a backup tool -- restic, borg,
borgmatic, duplicity, rsnapshot and the like -- started by a systemd timer, a cron
line or by hand. One round trip, as the login user (every process's command line
and cgroup are readable to it), reads the host's clock, the processes with their
age, and each one's cgroup; the systemd service a backup runs in is its job, the
service a timer starts, which is how netOrk shows the timer's scheduled job.
What a tool is doing is read from its subcommand: ``restic backup`` is a backup,
``restic forget`` a prune, ``restic mount`` none. ``borg serve`` counts: it is the
receiving end of another host's backup.
A ``ps`` without ``etimes`` (busybox) fails the reading rather than report nothing.
"""
from __future__ import annotations
import posixpath
import re
from typing import TYPE_CHECKING, NamedTuple
from napalm_device_types import RunningBackupDict, RunningBackupKind, RunningBackupsMixin
#: ``NOW <epoch>``, one ``pid<TAB>elapsed<TAB>cgroup<TAB>args`` line per process, ``END``.
RUNNING_BACKUPS_COMMAND = (
'echo "NOW $(date +%s)"; '
"out=$(LC_ALL=C ps -eo pid=,etimes=,args= 2>/dev/null) || exit 1; "
"printf '%s\\n' \"$out\" | while read -r pid et args; do "
'printf \'%s\\t%s\\t%s\\t%s\\n\' "$pid" "$et" '
'"$(tail -n 1 /proc/$pid/cgroup 2>/dev/null)" "$args"; '
"done; echo END"
)
_READ_TIMEOUT = 30
class _Tool(NamedTuple):
#: subcommand -> what it does; None: not a backup (a mount, a restore, a listing).
subcommands: dict[str, RunningBackupKind | None]
#: What the tool does when no listed subcommand is given; None: nothing to stop for.
default: RunningBackupKind | None = None
_RESTIC: dict[str, RunningBackupKind | None] = {
"backup": "backup",
"forget": "prune",
"prune": "prune",
"check": "verify",
"copy": "sync",
}
_BORG: dict[str, RunningBackupKind | None] = {
"create": "backup",
"serve": "backup",
"prune": "prune",
"compact": "prune",
"check": "verify",
}
_TOOLS: dict[str, _Tool] = {
"restic": _Tool(_RESTIC),
"resticprofile": _Tool(_RESTIC),
"autorestic": _Tool({"backup": "backup", "forget": "prune", "check": "verify"}),
"borg": _Tool(_BORG),
"borgbackup": _Tool(_BORG),
"borgmatic": _Tool(
{"extract": None, "list": None, "info": None, "restore": None, "mount": None},
"backup",
),
"duplicity": _Tool(
{
"restore": None,
"collection-status": None,
"list-current-files": None,
"verify": "verify",
"cleanup": "prune",
"remove-older-than": "prune",
"remove-all-but-n-full": "prune",
"remove-all-inc-of-but-n-full": "prune",
},
"backup",
),
"duply": _Tool(
{
"backup": "backup",
"bkp": "backup",
"incr": "backup",
"full": "backup",
"purge": "prune",
"verify": "verify",
}
),
"rsnapshot": _Tool({}, "backup"),
"rdiff-backup": _Tool(
{"-r": None, "--restore-as-of": None, "restore": None, "list": None}, "backup"
),
"proxmox-backup-client": _Tool({"backup": "backup"}),
"kopia": _Tool({"snapshot": "backup", "maintenance": "prune"}),
"btrbk": _Tool({"run": "backup", "resume": "backup", "snapshot": "backup", "prune": "prune"}),
}
#: What may stand before the tool on its command line: an interpreter or a wrapper,
#: their options and plain numbers or durations ("nice -n 19", "timeout 3h").
_WRAPPERS = frozenset(
{"sh", "bash", "dash", "env", "nice", "ionice", "nocache", "timeout", "chrt", "perl"}
)
_PYTHON = re.compile(r"python[0-9.]*")
_ARGUMENT = re.compile(r"-.*|\d+[smhd]?")
#: Services that start other people's commands: a backup run from them has no job of its own.
_NOT_JOBS = frozenset(
{"cron.service", "crond.service", "cronie.service", "anacron.service", "atd.service"}
| {"ssh.service", "sshd.service"}
)
def _tool(args: str) -> tuple[_Tool, list[str]] | None:
"""The backup tool *args* runs and its arguments, or None for any other process."""
words = args.split()
for index, word in enumerate(words):
name = posixpath.basename(word)
if name in _TOOLS:
return _TOOLS[name], words[index + 1 :]
if not (name in _WRAPPERS or _PYTHON.fullmatch(name) or _ARGUMENT.fullmatch(word)):
return None
return None
def backup_kind(args: str) -> RunningBackupKind | None:
"""What the process with command line *args* is doing for a backup, or None."""
found = _tool(args)
if found is None:
return None
tool, rest = found
for word in rest:
if word in tool.subcommands:
return tool.subcommands[word]
return tool.default
def _job(cgroup: str) -> str | None:
"""The systemd service a process runs in, when that service is its own job."""
unit = posixpath.basename(cgroup.rsplit(":", 1)[-1])
return unit if unit.endswith(".service") and unit not in _NOT_JOBS else None
def _clock(lines: list[str]) -> int:
if not lines or not lines[0].startswith("NOW "):
raise RuntimeError("get_running_backups: the host did not say what time it is")
try:
return int(lines[0].split()[1])
except (IndexError, ValueError) as exc:
raise RuntimeError(f"get_running_backups: not a time: {lines[0]!r}") from exc
def parse_running_backups(output: str) -> list[RunningBackupDict]:
"""The backups in ``RUNNING_BACKUPS_COMMAND``'s *output*.
:raises RuntimeError: when the reading was cut short or never had the clock.
"""
lines = output.splitlines()
now = _clock(lines)
if lines[-1].strip() != "END":
raise RuntimeError("get_running_backups: the process list was cut short")
found: list[RunningBackupDict] = []
for line in lines[1:-1]:
parts = line.split("\t", 3)
if len(parts) != 4 or not parts[1].isdigit():
continue
pid, elapsed, cgroup, args = parts
kind = backup_kind(args)
if kind is not None:
since = now - int(elapsed)
found.append(
{"kind": kind, "job": _job(cgroup), "task": pid, "guests": [], "since": since}
)
return found
class LinuxRunningBackupsMixin(RunningBackupsMixin):
"""``get_running_backups`` over the driver's own session, as the login user."""
if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers
def _send(self, command: str, read_timeout: float = 100) -> str: ...
def get_running_backups(self) -> list[RunningBackupDict]:
return parse_running_backups(self._send(RUNNING_BACKUPS_COMMAND, _READ_TIMEOUT))