feat(running-backups): the host's own backups a restart would abort #73
No files matched your search
+8
-1
@@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- `get_running_backups()` through napalm-device-types' `RunningBackupsMixin` (4.34.0): the
|
||||
host's own backups a restart would abort -- a process of restic, resticprofile, autorestic,
|
||||
borg, borgmatic, duplicity, duply, rsnapshot, rdiff-backup, proxmox-backup-client, kopia or
|
||||
btrbk doing a backup, prune, check or copy (not a mount, a restore or a listing), with the
|
||||
systemd service it runs in as its job (a timer's service; none for a cron or a login
|
||||
session), its process id and when it started. One round trip as the login user; a `ps`
|
||||
without `etimes` fails the reading (NetOrk/netork#2318). OpenMediaVault and QNAP inherit it.
|
||||
- `get_log_sources()` through napalm-device-types' `LogSourcesMixin` (4.30.0): the units
|
||||
journald holds entries for and the log files running processes hold open, with the service
|
||||
or container writing each, as root, and without root what the login user may see
|
||||
@@ -18,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
admin groups. The account stays open and logs in with its other keys only.
|
||||
|
||||
### Changed
|
||||
- Needs napalm-device-types 4.29.0.
|
||||
- Needs napalm-device-types 4.34.0.
|
||||
|
||||
## [0.17.0] - 2026-11-04
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Readings that need root to see every process: listening sockets and log sources.
|
||||
"""Readings of the host's processes: listening sockets, log sources, running backups.
|
||||
|
||||
Both commands are napalm-device-types' and read-only. Root only widens what they
|
||||
see -- ``ss`` names every process behind a socket, ``/proc/<pid>/fd`` shows every
|
||||
The first two commands are napalm-device-types' and read-only. Root only widens
|
||||
what they see -- ``ss`` names every process behind a socket, ``/proc/<pid>/fd`` shows every
|
||||
process's open files -- so both go through sudo where the driver can, and as the
|
||||
login user where it cannot; the mixins then say the reading is not ``attributed``.
|
||||
The running backups need no root: every process's command line and cgroup are
|
||||
readable to the login user (``_running_backups``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -12,12 +14,15 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from napalm_device_types import ListeningSocketsMixin, LogSourcesMixin
|
||||
|
||||
from napalm_linux._running_backups import LinuxRunningBackupsMixin
|
||||
|
||||
#: Seconds either reading may take; a busy host lists thousands of descriptors.
|
||||
_READ_TIMEOUT = 60
|
||||
|
||||
|
||||
class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin):
|
||||
"""The transports for ``get_listening_sockets`` and ``get_log_sources``."""
|
||||
class LinuxProcessReadingsMixin(ListeningSocketsMixin, LogSourcesMixin, LinuxRunningBackupsMixin):
|
||||
"""The transports for ``get_listening_sockets`` and ``get_log_sources``, and
|
||||
``get_running_backups``."""
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""The host's own backups a restart would abort (napalm-device-types' ``RunningBackupsMixin``).
|
||||
|
||||
A file-level backup on a Linux host is a process of a backup tool -- restic, borg,
|
||||
borgmatic, duplicity, rsnapshot and the like -- started by a systemd timer, a cron
|
||||
line or by hand. One round trip, as the login user (every process's command line
|
||||
and cgroup are readable to it), reads the host's clock, the processes with their
|
||||
age, and each one's cgroup; the systemd service a backup runs in is its job, the
|
||||
service a timer starts, which is how netOrk shows the timer's scheduled job.
|
||||
|
||||
What a tool is doing is read from its subcommand: ``restic backup`` is a backup,
|
||||
``restic forget`` a prune, ``restic mount`` none. ``borg serve`` counts: it is the
|
||||
receiving end of another host's backup.
|
||||
|
||||
A ``ps`` without ``etimes`` (busybox) fails the reading rather than report nothing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import posixpath
|
||||
import re
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from napalm_device_types import RunningBackupDict, RunningBackupKind, RunningBackupsMixin
|
||||
|
||||
#: ``NOW <epoch>``, one ``pid<TAB>elapsed<TAB>cgroup<TAB>args`` line per process, ``END``.
|
||||
RUNNING_BACKUPS_COMMAND = (
|
||||
'echo "NOW $(date +%s)"; '
|
||||
"out=$(LC_ALL=C ps -eo pid=,etimes=,args= 2>/dev/null) || exit 1; "
|
||||
"printf '%s\\n' \"$out\" | while read -r pid et args; do "
|
||||
'printf \'%s\\t%s\\t%s\\t%s\\n\' "$pid" "$et" '
|
||||
'"$(tail -n 1 /proc/$pid/cgroup 2>/dev/null)" "$args"; '
|
||||
"done; echo END"
|
||||
)
|
||||
|
||||
_READ_TIMEOUT = 30
|
||||
|
||||
|
||||
class _Tool(NamedTuple):
|
||||
#: subcommand -> what it does; None: not a backup (a mount, a restore, a listing).
|
||||
subcommands: dict[str, RunningBackupKind | None]
|
||||
#: What the tool does when no listed subcommand is given; None: nothing to stop for.
|
||||
default: RunningBackupKind | None = None
|
||||
|
||||
|
||||
_RESTIC: dict[str, RunningBackupKind | None] = {
|
||||
"backup": "backup",
|
||||
"forget": "prune",
|
||||
"prune": "prune",
|
||||
"check": "verify",
|
||||
"copy": "sync",
|
||||
}
|
||||
_BORG: dict[str, RunningBackupKind | None] = {
|
||||
"create": "backup",
|
||||
"serve": "backup",
|
||||
"prune": "prune",
|
||||
"compact": "prune",
|
||||
"check": "verify",
|
||||
}
|
||||
|
||||
_TOOLS: dict[str, _Tool] = {
|
||||
"restic": _Tool(_RESTIC),
|
||||
"resticprofile": _Tool(_RESTIC),
|
||||
"autorestic": _Tool({"backup": "backup", "forget": "prune", "check": "verify"}),
|
||||
"borg": _Tool(_BORG),
|
||||
"borgbackup": _Tool(_BORG),
|
||||
"borgmatic": _Tool(
|
||||
{"extract": None, "list": None, "info": None, "restore": None, "mount": None},
|
||||
"backup",
|
||||
),
|
||||
"duplicity": _Tool(
|
||||
{
|
||||
"restore": None,
|
||||
"collection-status": None,
|
||||
"list-current-files": None,
|
||||
"verify": "verify",
|
||||
"cleanup": "prune",
|
||||
"remove-older-than": "prune",
|
||||
"remove-all-but-n-full": "prune",
|
||||
"remove-all-inc-of-but-n-full": "prune",
|
||||
},
|
||||
"backup",
|
||||
),
|
||||
"duply": _Tool(
|
||||
{
|
||||
"backup": "backup",
|
||||
"bkp": "backup",
|
||||
"incr": "backup",
|
||||
"full": "backup",
|
||||
"purge": "prune",
|
||||
"verify": "verify",
|
||||
}
|
||||
),
|
||||
"rsnapshot": _Tool({}, "backup"),
|
||||
"rdiff-backup": _Tool(
|
||||
{"-r": None, "--restore-as-of": None, "restore": None, "list": None}, "backup"
|
||||
),
|
||||
"proxmox-backup-client": _Tool({"backup": "backup"}),
|
||||
"kopia": _Tool({"snapshot": "backup", "maintenance": "prune"}),
|
||||
"btrbk": _Tool({"run": "backup", "resume": "backup", "snapshot": "backup", "prune": "prune"}),
|
||||
}
|
||||
|
||||
#: What may stand before the tool on its command line: an interpreter or a wrapper,
|
||||
#: their options and plain numbers or durations ("nice -n 19", "timeout 3h").
|
||||
_WRAPPERS = frozenset(
|
||||
{"sh", "bash", "dash", "env", "nice", "ionice", "nocache", "timeout", "chrt", "perl"}
|
||||
)
|
||||
_PYTHON = re.compile(r"python[0-9.]*")
|
||||
_ARGUMENT = re.compile(r"-.*|\d+[smhd]?")
|
||||
|
||||
#: Services that start other people's commands: a backup run from them has no job of its own.
|
||||
_NOT_JOBS = frozenset(
|
||||
{"cron.service", "crond.service", "cronie.service", "anacron.service", "atd.service"}
|
||||
| {"ssh.service", "sshd.service"}
|
||||
)
|
||||
|
||||
|
||||
def _tool(args: str) -> tuple[_Tool, list[str]] | None:
|
||||
"""The backup tool *args* runs and its arguments, or None for any other process."""
|
||||
words = args.split()
|
||||
for index, word in enumerate(words):
|
||||
name = posixpath.basename(word)
|
||||
if name in _TOOLS:
|
||||
return _TOOLS[name], words[index + 1 :]
|
||||
if not (name in _WRAPPERS or _PYTHON.fullmatch(name) or _ARGUMENT.fullmatch(word)):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def backup_kind(args: str) -> RunningBackupKind | None:
|
||||
"""What the process with command line *args* is doing for a backup, or None."""
|
||||
found = _tool(args)
|
||||
if found is None:
|
||||
return None
|
||||
tool, rest = found
|
||||
for word in rest:
|
||||
if word in tool.subcommands:
|
||||
return tool.subcommands[word]
|
||||
return tool.default
|
||||
|
||||
|
||||
def _job(cgroup: str) -> str | None:
|
||||
"""The systemd service a process runs in, when that service is its own job."""
|
||||
unit = posixpath.basename(cgroup.rsplit(":", 1)[-1])
|
||||
return unit if unit.endswith(".service") and unit not in _NOT_JOBS else None
|
||||
|
||||
|
||||
def _clock(lines: list[str]) -> int:
|
||||
if not lines or not lines[0].startswith("NOW "):
|
||||
raise RuntimeError("get_running_backups: the host did not say what time it is")
|
||||
try:
|
||||
return int(lines[0].split()[1])
|
||||
except (IndexError, ValueError) as exc:
|
||||
raise RuntimeError(f"get_running_backups: not a time: {lines[0]!r}") from exc
|
||||
|
||||
|
||||
def parse_running_backups(output: str) -> list[RunningBackupDict]:
|
||||
"""The backups in ``RUNNING_BACKUPS_COMMAND``'s *output*.
|
||||
|
||||
:raises RuntimeError: when the reading was cut short or never had the clock.
|
||||
"""
|
||||
lines = output.splitlines()
|
||||
now = _clock(lines)
|
||||
if lines[-1].strip() != "END":
|
||||
raise RuntimeError("get_running_backups: the process list was cut short")
|
||||
found: list[RunningBackupDict] = []
|
||||
for line in lines[1:-1]:
|
||||
parts = line.split("\t", 3)
|
||||
if len(parts) != 4 or not parts[1].isdigit():
|
||||
continue
|
||||
pid, elapsed, cgroup, args = parts
|
||||
kind = backup_kind(args)
|
||||
if kind is not None:
|
||||
since = now - int(elapsed)
|
||||
found.append(
|
||||
{"kind": kind, "job": _job(cgroup), "task": pid, "guests": [], "since": since}
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
class LinuxRunningBackupsMixin(RunningBackupsMixin):
|
||||
"""``get_running_backups`` over the driver's own session, as the login user."""
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - the driver's own, declared for type checkers
|
||||
|
||||
def _send(self, command: str, read_timeout: float = 100) -> str: ...
|
||||
|
||||
def get_running_backups(self) -> list[RunningBackupDict]:
|
||||
return parse_running_backups(self._send(RUNNING_BACKUPS_COMMAND, _READ_TIMEOUT))
|
||||
+1
-1
@@ -34,7 +34,7 @@ classifiers = [
|
||||
]
|
||||
dependencies = [
|
||||
"napalm>=4.0",
|
||||
"napalm-device-types>=4.30.0",
|
||||
"napalm-device-types>=4.34.0",
|
||||
"netmiko>=4.0.0",
|
||||
"paramiko>=5.0.0", # CVE-2026-44405
|
||||
]
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""get_running_backups: the host's own file-level backups that a restart would abort.
|
||||
|
||||
A restic, borg, borgmatic, duplicity ... run -- from a systemd timer, a cron line
|
||||
or by hand -- is a process of a backup tool. The driver reads the processes, the
|
||||
host's clock and each process's cgroup in one round trip, and names a backup by
|
||||
the systemd service it runs in, which is the scheduled job netOrk shows for the
|
||||
timer (NetOrk/netork#2318, #1933). A failed read raises: an empty list would say
|
||||
nothing runs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from napalm_linux import LinuxDriver
|
||||
from napalm_linux._running_backups import RUNNING_BACKUPS_COMMAND, parse_running_backups
|
||||
|
||||
NOW = 1793664000
|
||||
|
||||
|
||||
def _reading(*rows: str, now: int = NOW, end: bool = True) -> str:
|
||||
lines = [f"NOW {now}", *rows]
|
||||
if end:
|
||||
lines.append("END")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _row(pid: int, elapsed: int, cgroup: str, args: str) -> str:
|
||||
return f"{pid}\t{elapsed}\t{cgroup}\t{args}"
|
||||
|
||||
|
||||
def test_nothing_runs():
|
||||
out = _reading(
|
||||
_row(1, 99999, "0::/init.scope", "/sbin/init"),
|
||||
_row(812, 3600, "0::/system.slice/ssh.service", "sshd: /usr/sbin/sshd -D"),
|
||||
)
|
||||
assert parse_running_backups(out) == []
|
||||
|
||||
|
||||
def test_a_timers_restic_backup_is_named_by_its_service():
|
||||
out = _reading(
|
||||
_row(4242, 600, "0::/system.slice/restic-backup.service", "/usr/bin/restic backup /srv")
|
||||
)
|
||||
assert parse_running_backups(out) == [
|
||||
{
|
||||
"kind": "backup",
|
||||
"job": "restic-backup.service",
|
||||
"task": "4242",
|
||||
"guests": [],
|
||||
"since": NOW - 600,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_a_cron_run_has_no_job_of_its_own():
|
||||
out = _reading(_row(77, 30, "0::/system.slice/cron.service", "borg create ::{now} /etc"))
|
||||
[backup] = parse_running_backups(out)
|
||||
assert backup["job"] is None
|
||||
assert backup["kind"] == "backup"
|
||||
|
||||
|
||||
def test_cgroup_v1_is_read_too():
|
||||
out = _reading(_row(9, 5, "1:name=systemd:/system.slice/borgmatic.service", "borgmatic"))
|
||||
assert parse_running_backups(out)[0]["job"] == "borgmatic.service"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "kind"),
|
||||
[
|
||||
("/usr/bin/restic backup --files-from /etc/restic/files", "backup"),
|
||||
("restic forget --prune --keep-daily 7", "prune"),
|
||||
("restic -r s3:example check", "verify"),
|
||||
("restic copy --from-repo /srv/a", "sync"),
|
||||
("/usr/bin/borg create --stats ::{now} /home", "backup"),
|
||||
("borg --lock-wait 60 prune --keep-daily 7", "prune"),
|
||||
("borg compact", "prune"),
|
||||
("borg check --verify-data", "verify"),
|
||||
("borg serve --restrict-to-path /srv/borg", "backup"),
|
||||
("/usr/bin/python3 /usr/bin/borgmatic --verbosity 1", "backup"),
|
||||
("duplicity incremental /home file:///mnt/b", "backup"),
|
||||
("/bin/bash /usr/bin/duply home backup", "backup"),
|
||||
("rsnapshot daily", "backup"),
|
||||
("rdiff-backup /home /mnt/b", "backup"),
|
||||
("proxmox-backup-client backup root.pxar:/", "backup"),
|
||||
("kopia snapshot create /home", "backup"),
|
||||
("btrbk run", "backup"),
|
||||
("resticprofile backup", "backup"),
|
||||
("autorestic backup -a", "backup"),
|
||||
],
|
||||
)
|
||||
def test_the_tools_and_what_they_do(args, kind):
|
||||
[backup] = parse_running_backups(_reading(_row(1, 1, "0::/user.slice", args)))
|
||||
assert backup["kind"] == kind
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args",
|
||||
[
|
||||
"restic mount /mnt/restic",
|
||||
"restic snapshots",
|
||||
"restic restore latest --target /",
|
||||
"borg mount ::archive /mnt",
|
||||
"borg list",
|
||||
"kopia server start",
|
||||
"vim /etc/restic/backup.sh",
|
||||
"grep borg /var/log/syslog",
|
||||
],
|
||||
)
|
||||
def test_what_is_no_backup(args):
|
||||
assert parse_running_backups(_reading(_row(1, 1, "0::/user.slice", args))) == []
|
||||
|
||||
|
||||
def test_a_session_or_scope_is_no_job():
|
||||
out = _reading(_row(5, 1, "0::/user.slice/user-1000.slice/session-3.scope", "restic backup /"))
|
||||
assert parse_running_backups(out)[0]["job"] is None
|
||||
|
||||
|
||||
def test_a_reading_cut_short_raises():
|
||||
with pytest.raises(RuntimeError):
|
||||
parse_running_backups(_reading(_row(1, 1, "", "restic backup /"), end=False))
|
||||
|
||||
|
||||
def test_no_clock_raises():
|
||||
with pytest.raises(RuntimeError):
|
||||
parse_running_backups("END")
|
||||
|
||||
|
||||
def test_the_driver_reads_as_the_login_user_in_one_round_trip():
|
||||
drv = LinuxDriver.__new__(LinuxDriver)
|
||||
drv._send = MagicMock(return_value=_reading())
|
||||
assert drv.get_running_backups() == []
|
||||
drv._send.assert_called_once()
|
||||
assert drv._send.call_args.args[0] == RUNNING_BACKUPS_COMMAND
|
||||
|
||||
|
||||
def test_the_command_says_when_ps_failed():
|
||||
"""Busybox's ps knows no etimes: the reading must not end as if nothing ran."""
|
||||
assert "|| exit" in RUNNING_BACKUPS_COMMAND
|
||||
assert RUNNING_BACKUPS_COMMAND.rstrip().endswith("echo END")
|
||||
Reference in new issue
Block a user