From 78374cfb420b5720c1846d0f1ad902ff10d316c8 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 10 Nov 2026 23:45:49 +0100 Subject: [PATCH] feat(running-backups): the host's own backups a restart would abort netOrk holds a host's restart after patching while a backup runs on it (NetOrk/netork#2318) and asks right before the restart. On a Linux host a file-level backup is a process of a backup tool, started by a timer, a cron line or by hand, so get_running_backups() (napalm-device-types 4.34.0) reads the processes, the clock and each process's cgroup in one round trip as the login user. What a tool does comes from its subcommand: restic backup is a backup, restic forget a prune, restic mount none; borg serve counts, as the receiving end of another host's backup. The systemd service a backup runs in is its job, the timer's service netOrk shows as its scheduled job. A ps without etimes (busybox) fails the reading rather than report nothing. --- CHANGELOG.md | 9 +- napalm_linux/_readings.py | 15 ++- napalm_linux/_running_backups.py | 188 +++++++++++++++++++++++++++++++ pyproject.toml | 2 +- tests/test_running_backups.py | 141 +++++++++++++++++++++++ 5 files changed, 348 insertions(+), 7 deletions(-) create mode 100644 napalm_linux/_running_backups.py create mode 100644 tests/test_running_backups.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fe28a6..897a950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `get_running_backups()` through napalm-device-types' `RunningBackupsMixin` (4.34.0): the + host's own backups a restart would abort -- a process of restic, resticprofile, autorestic, + borg, borgmatic, duplicity, duply, rsnapshot, rdiff-backup, proxmox-backup-client, kopia or + btrbk doing a backup, prune, check or copy (not a mount, a restore or a listing), with the + systemd service it runs in as its job (a timer's service; none for a cron or a login + session), its process id and when it started. One round trip as the login user; a `ps` + without `etimes` fails the reading (NetOrk/netork#2318). OpenMediaVault and QNAP inherit it. - `get_log_sources()` through napalm-device-types' `LogSourcesMixin` (4.30.0): the units journald holds entries for and the log files running processes hold open, with the service or container writing each, as root, and without root what the login user may see @@ -18,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 admin groups. The account stays open and logs in with its other keys only. ### Changed -- Needs napalm-device-types 4.29.0. +- Needs napalm-device-types 4.34.0. ## [0.17.0] - 2026-11-04 diff --git a/napalm_linux/_readings.py b/napalm_linux/_readings.py index 2db711e..42f3360 100644 --- a/napalm_linux/_readings.py +++ b/napalm_linux/_readings.py @@ -1,9 +1,11 @@ -"""Readings that need root to see every process: listening sockets and log sources. +"""Readings of the host's processes: listening sockets, log sources, running backups. -Both commands are napalm-device-types' and read-only. Root only widens what they -see -- ``ss`` names every process behind a socket, ``/proc//fd`` shows every +The first two commands are napalm-device-types' and read-only. Root only widens +what they see -- ``ss`` names every process behind a socket, ``/proc//fd`` shows every process's open files -- so both go through sudo where the driver can, and as the login user where it cannot; the mixins then say the reading is not ``attributed``. +The running backups need no root: every process's command line and cgroup are +readable to the login user (``_running_backups``). """ from __future__ import annotations @@ -12,12 +14,15 @@ from typing import TYPE_CHECKING from napalm_device_types import ListeningSocketsMixin, LogSourcesMixin +from napalm_linux._running_backups import LinuxRunningBackupsMixin + #: Seconds either reading may take; a busy host lists thousands of descriptors. _READ_TIMEOUT = 60 -class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin): - """The transports for ``get_listening_sockets`` and ``get_log_sources``.""" +class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin, LinuxRunningBackupsMixin): + """The transports for ``get_listening_sockets`` and ``get_log_sources``, and + ``get_running_backups``.""" if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers diff --git a/napalm_linux/_running_backups.py b/napalm_linux/_running_backups.py new file mode 100644 index 0000000..1235b4a --- /dev/null +++ b/napalm_linux/_running_backups.py @@ -0,0 +1,188 @@ +"""The host's own backups a restart would abort (napalm-device-types' ``RunningBackupsMixin``). + +A file-level backup on a Linux host is a process of a backup tool -- restic, borg, +borgmatic, duplicity, rsnapshot and the like -- started by a systemd timer, a cron +line or by hand. One round trip, as the login user (every process's command line +and cgroup are readable to it), reads the host's clock, the processes with their +age, and each one's cgroup; the systemd service a backup runs in is its job, the +service a timer starts, which is how netOrk shows the timer's scheduled job. + +What a tool is doing is read from its subcommand: ``restic backup`` is a backup, +``restic forget`` a prune, ``restic mount`` none. ``borg serve`` counts: it is the +receiving end of another host's backup. + +A ``ps`` without ``etimes`` (busybox) fails the reading rather than report nothing. +""" + +from __future__ import annotations + +import posixpath +import re +from typing import TYPE_CHECKING, NamedTuple + +from napalm_device_types import RunningBackupDict, RunningBackupKind, RunningBackupsMixin + +#: ``NOW ``, one ``pidelapsedcgroupargs`` line per process, ``END``. +RUNNING_BACKUPS_COMMAND = ( + 'echo "NOW $(date +%s)"; ' + "out=$(LC_ALL=C ps -eo pid=,etimes=,args= 2>/dev/null) || exit 1; " + "printf '%s\\n' \"$out\" | while read -r pid et args; do " + 'printf \'%s\\t%s\\t%s\\t%s\\n\' "$pid" "$et" ' + '"$(tail -n 1 /proc/$pid/cgroup 2>/dev/null)" "$args"; ' + "done; echo END" +) + +_READ_TIMEOUT = 30 + + +class _Tool(NamedTuple): + #: subcommand -> what it does; None: not a backup (a mount, a restore, a listing). + subcommands: dict[str, RunningBackupKind | None] + #: What the tool does when no listed subcommand is given; None: nothing to stop for. + default: RunningBackupKind | None = None + + +_RESTIC: dict[str, RunningBackupKind | None] = { + "backup": "backup", + "forget": "prune", + "prune": "prune", + "check": "verify", + "copy": "sync", +} +_BORG: dict[str, RunningBackupKind | None] = { + "create": "backup", + "serve": "backup", + "prune": "prune", + "compact": "prune", + "check": "verify", +} + +_TOOLS: dict[str, _Tool] = { + "restic": _Tool(_RESTIC), + "resticprofile": _Tool(_RESTIC), + "autorestic": _Tool({"backup": "backup", "forget": "prune", "check": "verify"}), + "borg": _Tool(_BORG), + "borgbackup": _Tool(_BORG), + "borgmatic": _Tool( + {"extract": None, "list": None, "info": None, "restore": None, "mount": None}, + "backup", + ), + "duplicity": _Tool( + { + "restore": None, + "collection-status": None, + "list-current-files": None, + "verify": "verify", + "cleanup": "prune", + "remove-older-than": "prune", + "remove-all-but-n-full": "prune", + "remove-all-inc-of-but-n-full": "prune", + }, + "backup", + ), + "duply": _Tool( + { + "backup": "backup", + "bkp": "backup", + "incr": "backup", + "full": "backup", + "purge": "prune", + "verify": "verify", + } + ), + "rsnapshot": _Tool({}, "backup"), + "rdiff-backup": _Tool( + {"-r": None, "--restore-as-of": None, "restore": None, "list": None}, "backup" + ), + "proxmox-backup-client": _Tool({"backup": "backup"}), + "kopia": _Tool({"snapshot": "backup", "maintenance": "prune"}), + "btrbk": _Tool({"run": "backup", "resume": "backup", "snapshot": "backup", "prune": "prune"}), +} + +#: What may stand before the tool on its command line: an interpreter or a wrapper, +#: their options and plain numbers or durations ("nice -n 19", "timeout 3h"). +_WRAPPERS = frozenset( + {"sh", "bash", "dash", "env", "nice", "ionice", "nocache", "timeout", "chrt", "perl"} +) +_PYTHON = re.compile(r"python[0-9.]*") +_ARGUMENT = re.compile(r"-.*|\d+[smhd]?") + +#: Services that start other people's commands: a backup run from them has no job of its own. +_NOT_JOBS = frozenset( + {"cron.service", "crond.service", "cronie.service", "anacron.service", "atd.service"} + | {"ssh.service", "sshd.service"} +) + + +def _tool(args: str) -> tuple[_Tool, list[str]] | None: + """The backup tool *args* runs and its arguments, or None for any other process.""" + words = args.split() + for index, word in enumerate(words): + name = posixpath.basename(word) + if name in _TOOLS: + return _TOOLS[name], words[index + 1 :] + if not (name in _WRAPPERS or _PYTHON.fullmatch(name) or _ARGUMENT.fullmatch(word)): + return None + return None + + +def backup_kind(args: str) -> RunningBackupKind | None: + """What the process with command line *args* is doing for a backup, or None.""" + found = _tool(args) + if found is None: + return None + tool, rest = found + for word in rest: + if word in tool.subcommands: + return tool.subcommands[word] + return tool.default + + +def _job(cgroup: str) -> str | None: + """The systemd service a process runs in, when that service is its own job.""" + unit = posixpath.basename(cgroup.rsplit(":", 1)[-1]) + return unit if unit.endswith(".service") and unit not in _NOT_JOBS else None + + +def _clock(lines: list[str]) -> int: + if not lines or not lines[0].startswith("NOW "): + raise RuntimeError("get_running_backups: the host did not say what time it is") + try: + return int(lines[0].split()[1]) + except (IndexError, ValueError) as exc: + raise RuntimeError(f"get_running_backups: not a time: {lines[0]!r}") from exc + + +def parse_running_backups(output: str) -> list[RunningBackupDict]: + """The backups in ``RUNNING_BACKUPS_COMMAND``'s *output*. + + :raises RuntimeError: when the reading was cut short or never had the clock. + """ + lines = output.splitlines() + now = _clock(lines) + if lines[-1].strip() != "END": + raise RuntimeError("get_running_backups: the process list was cut short") + found: list[RunningBackupDict] = [] + for line in lines[1:-1]: + parts = line.split("\t", 3) + if len(parts) != 4 or not parts[1].isdigit(): + continue + pid, elapsed, cgroup, args = parts + kind = backup_kind(args) + if kind is not None: + since = now - int(elapsed) + found.append( + {"kind": kind, "job": _job(cgroup), "task": pid, "guests": [], "since": since} + ) + return found + + +class LinuxRunningBackupsMixin(RunningBackupsMixin): + """``get_running_backups`` over the driver's own session, as the login user.""" + + if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers + + def _send(self, command: str, read_timeout: float = 100) -> str: ... + + def get_running_backups(self) -> list[RunningBackupDict]: + return parse_running_backups(self._send(RUNNING_BACKUPS_COMMAND, _READ_TIMEOUT)) diff --git a/pyproject.toml b/pyproject.toml index 2599dfb..2cbed6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ classifiers = [ ] dependencies = [ "napalm>=4.0", - "napalm-device-types>=4.30.0", + "napalm-device-types>=4.34.0", "netmiko>=4.0.0", "paramiko>=5.0.0", # CVE-2026-44405 ] diff --git a/tests/test_running_backups.py b/tests/test_running_backups.py new file mode 100644 index 0000000..cc87c7a --- /dev/null +++ b/tests/test_running_backups.py @@ -0,0 +1,141 @@ +"""get_running_backups: the host's own file-level backups that a restart would abort. + +A restic, borg, borgmatic, duplicity ... run -- from a systemd timer, a cron line +or by hand -- is a process of a backup tool. The driver reads the processes, the +host's clock and each process's cgroup in one round trip, and names a backup by +the systemd service it runs in, which is the scheduled job netOrk shows for the +timer (NetOrk/netork#2318, #1933). A failed read raises: an empty list would say +nothing runs. +""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from napalm_linux import LinuxDriver +from napalm_linux._running_backups import RUNNING_BACKUPS_COMMAND, parse_running_backups + +NOW = 1793664000 + + +def _reading(*rows: str, now: int = NOW, end: bool = True) -> str: + lines = [f"NOW {now}", *rows] + if end: + lines.append("END") + return "\n".join(lines) + + +def _row(pid: int, elapsed: int, cgroup: str, args: str) -> str: + return f"{pid}\t{elapsed}\t{cgroup}\t{args}" + + +def test_nothing_runs(): + out = _reading( + _row(1, 99999, "0::/init.scope", "/sbin/init"), + _row(812, 3600, "0::/system.slice/ssh.service", "sshd: /usr/sbin/sshd -D"), + ) + assert parse_running_backups(out) == [] + + +def test_a_timers_restic_backup_is_named_by_its_service(): + out = _reading( + _row(4242, 600, "0::/system.slice/restic-backup.service", "/usr/bin/restic backup /srv") + ) + assert parse_running_backups(out) == [ + { + "kind": "backup", + "job": "restic-backup.service", + "task": "4242", + "guests": [], + "since": NOW - 600, + } + ] + + +def test_a_cron_run_has_no_job_of_its_own(): + out = _reading(_row(77, 30, "0::/system.slice/cron.service", "borg create ::{now} /etc")) + [backup] = parse_running_backups(out) + assert backup["job"] is None + assert backup["kind"] == "backup" + + +def test_cgroup_v1_is_read_too(): + out = _reading(_row(9, 5, "1:name=systemd:/system.slice/borgmatic.service", "borgmatic")) + assert parse_running_backups(out)[0]["job"] == "borgmatic.service" + + +@pytest.mark.parametrize( + ("args", "kind"), + [ + ("/usr/bin/restic backup --files-from /etc/restic/files", "backup"), + ("restic forget --prune --keep-daily 7", "prune"), + ("restic -r s3:example check", "verify"), + ("restic copy --from-repo /srv/a", "sync"), + ("/usr/bin/borg create --stats ::{now} /home", "backup"), + ("borg --lock-wait 60 prune --keep-daily 7", "prune"), + ("borg compact", "prune"), + ("borg check --verify-data", "verify"), + ("borg serve --restrict-to-path /srv/borg", "backup"), + ("/usr/bin/python3 /usr/bin/borgmatic --verbosity 1", "backup"), + ("duplicity incremental /home file:///mnt/b", "backup"), + ("/bin/bash /usr/bin/duply home backup", "backup"), + ("rsnapshot daily", "backup"), + ("rdiff-backup /home /mnt/b", "backup"), + ("proxmox-backup-client backup root.pxar:/", "backup"), + ("kopia snapshot create /home", "backup"), + ("btrbk run", "backup"), + ("resticprofile backup", "backup"), + ("autorestic backup -a", "backup"), + ], +) +def test_the_tools_and_what_they_do(args, kind): + [backup] = parse_running_backups(_reading(_row(1, 1, "0::/user.slice", args))) + assert backup["kind"] == kind + + +@pytest.mark.parametrize( + "args", + [ + "restic mount /mnt/restic", + "restic snapshots", + "restic restore latest --target /", + "borg mount ::archive /mnt", + "borg list", + "kopia server start", + "vim /etc/restic/backup.sh", + "grep borg /var/log/syslog", + ], +) +def test_what_is_no_backup(args): + assert parse_running_backups(_reading(_row(1, 1, "0::/user.slice", args))) == [] + + +def test_a_session_or_scope_is_no_job(): + out = _reading(_row(5, 1, "0::/user.slice/user-1000.slice/session-3.scope", "restic backup /")) + assert parse_running_backups(out)[0]["job"] is None + + +def test_a_reading_cut_short_raises(): + with pytest.raises(RuntimeError): + parse_running_backups(_reading(_row(1, 1, "", "restic backup /"), end=False)) + + +def test_no_clock_raises(): + with pytest.raises(RuntimeError): + parse_running_backups("END") + + +def test_the_driver_reads_as_the_login_user_in_one_round_trip(): + drv = LinuxDriver.__new__(LinuxDriver) + drv._send = MagicMock(return_value=_reading()) + assert drv.get_running_backups() == [] + drv._send.assert_called_once() + assert drv._send.call_args.args[0] == RUNNING_BACKUPS_COMMAND + + +def test_the_command_says_when_ps_failed(): + """Busybox's ps knows no etimes: the reading must not end as if nothing ran.""" + assert "|| exit" in RUNNING_BACKUPS_COMMAND + assert RUNNING_BACKUPS_COMMAND.rstrip().endswith("echo END")