From c9472501798fce1f2b45d061fb45fcb1885fcdc0 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 11 Oct 2026 09:06:00 +0200 Subject: [PATCH] feat: report how full /, /boot and /var are in the host status A Proxmox host's root filesystem stood at 94 % with about 400 MB free for a week; a kernel upgrade failed on it with "No space left on device" and left dpkg halfway, and nothing had said the disk was full (NetOrk/netork#862). HOST_STATUS_COMMAND now also runs POSIX `df -Pk / /boot /var`, without root. get_host_status() returns `filesystems`: each filesystem once (a directory that is not a mount of its own is the filesystem it lives on), with its mountpoint, total, used and available bytes, or None when df said nothing. `available` is what df says an ordinary user can still write, which on ext4 is less than total minus used. How full is too full is the caller's decision. Version 4.4.0. --- README.md | 3 +- napalm_device_types/host_status.py | 39 ++++++++++++++- napalm_device_types/models.py | 15 ++++++ pyproject.toml | 2 +- tests/test_host_status.py | 78 ++++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e8cd863..fab1d1f 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,8 @@ a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, kernel of the running flavour installed), whether it patches itself (unattended-upgrades, dnf-automatic) and when that last succeeded (apt's `upgrade-stamp`), and whether dpkg was left halfway (a journal in `/var/lib/dpkg/updates` or a half-configured package; unknown -while a package manager is running). `package_updates` holds the shared apt and dnf parsers: apt's suites become +while a package manager is running), and how full `/`, `/boot` and `/var` are (`df -Pk`, +each filesystem once, in bytes; how full is too full is the caller's decision). `package_updates` holds the shared apt and dnf parsers: apt's suites become an update's `origin`, a `-security` suite makes it a security update, and dnf's security advisories do the same. diff --git a/napalm_device_types/host_status.py b/napalm_device_types/host_status.py index c96c8cc..82f4ea9 100644 --- a/napalm_device_types/host_status.py +++ b/napalm_device_types/host_status.py @@ -41,6 +41,15 @@ unattended-upgrade is running, packages are halfway on purpose -- also between two of apt's dpkg calls -- so the answer is then ``None``, as on a host without dpkg. unattended-upgrade is told from Ubuntu's idle shutdown helper by its command line (:data:`BUSY_CHECK`). + +**Filesystems** are how full ``/``, ``/boot`` and ``/var`` are, from POSIX +``df -Pk``: where an upgrade unpacks a kernel, builds its initrd and keeps the +package files. A kernel upgrade that ran out of room there left dpkg halfway on +a Proxmox host whose root had been 94 % full for a week (NetOrk/netork#862). A +directory that is not a mount of its own is reported as the filesystem it lives +on, so each filesystem appears once. ``available`` is what df says an ordinary +user can still write, which on ext4 is less than total minus used. How full is +too full is not decided here. """ from __future__ import annotations @@ -48,7 +57,11 @@ from __future__ import annotations import re from typing import Dict, List, Optional, Tuple, TYPE_CHECKING -from napalm_device_types.models import HostStatusDict, InterruptedPackageDict +from napalm_device_types.models import ( + FilesystemUsageDict, + HostStatusDict, + InterruptedPackageDict, +) from napalm_device_types.terminal import strip_terminal_codes _BEGIN = "HSTAT_BEGIN" @@ -58,6 +71,8 @@ _APT_TIMER = "apt-daily-upgrade.timer" _DNF_TIMERS = ("dnf-automatic.timer", "dnf-automatic-install.timer") _UPGRADE_STAMP = "/var/lib/apt/periodic/upgrade-stamp" _DPKG_JOURNAL = "/var/lib/dpkg/updates" +#: Where an upgrade writes: the kernel's modules, its initrd, the package files. +_FILESYSTEMS = "/ /boot /var" #: Prints ``[dpkg-busy]`` while a process that leaves packages halfway is working: #: dpkg, apt, apt-get or aptitude by name, unattended-upgrade by its command line. #: Its name is cut to 15 characters, ``unattended-upgr``, the same as that of @@ -91,6 +106,7 @@ HOST_STATUS_COMMAND = ( f"echo '[dpkg-journal]'; ls -1 {_DPKG_JOURNAL} 2>/dev/null | head -n 20; " "echo '[dpkg-audit]'; dpkg-query -W -f='${db:Status-Abbrev} ${Package}\\n' 2>/dev/null " "| awk 'substr($0, 2, 1) ~ /[HUFWt]/ || substr($0, 3, 1) == \"R\"' | head -n 50; fi; " + f"echo '[filesystems]'; df -Pk {_FILESYSTEMS}; " f"echo '[timers]'; for u in {_APT_TIMER} {' '.join(_DNF_TIMERS)}; do " 'printf \'%s %s\\n\' "$u" "$(systemctl is-enabled "$u" 2>/dev/null)"; done; ' "printf '%s%s\\n' HSTAT_ END; } 2>/dev/null | cat" @@ -101,6 +117,9 @@ _OFF_STATES = frozenset({"disabled", "masked"}) _INTERVAL = re.compile(r"(\d+)([smhd]?)") _UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "": 86400} _JOURNAL_ENTRY = re.compile(r"\d+") +#: One line of ``df -P``: device, 1024-blocks, used, available, capacity, mountpoint. +#: The device and the mountpoint may hold blanks; the four numbers between them do not. +_DF_LINE = re.compile(r"^.+?\s+(\d+)\s+(\d+)\s+(\d+)\s+(?:\d+%|-)\s+(/.*)$") #: dpkg's second status letter -> its word, for the states a cut-off run leaves. _HALFWAY = { "H": "half-installed", @@ -236,6 +255,22 @@ def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[Interrup return bool(packages) or journal, packages +def _filesystems(sections: Dict[str, List[str]]) -> Optional[List[FilesystemUsageDict]]: + """Each filesystem df named, once and in its order; None when it named none.""" + found: Dict[str, FilesystemUsageDict] = {} + for line in sections.get("filesystems") or []: + match = _DF_LINE.match(line) + if not match or int(match.group(1)) == 0: + continue + total, used, available = (int(match.group(i)) * 1024 for i in (1, 2, 3)) + mountpoint = match.group(4) + found.setdefault( + mountpoint, + {"mountpoint": mountpoint, "total": total, "used": used, "available": available}, + ) + return list(found.values()) or None + + def parse_host_status(output: str) -> HostStatusDict: """Parse what :data:`HOST_STATUS_COMMAND` printed. @@ -252,6 +287,7 @@ def parse_host_status(output: str) -> HostStatusDict: "auto_updates_interval": _interval(sections), "package_manager_interrupted": interrupted, "interrupted_packages": packages, + "filesystems": _filesystems(sections), } @@ -282,6 +318,7 @@ class HostStatusMixin: * auto_updates_interval (seconds or None) * package_manager_interrupted (bool or None) * interrupted_packages (list of {name, state}) + * filesystems (list of {mountpoint, total, used, available} in bytes, or None) :raises ValueError: if the host's output carried no intact report. """ diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index d17dcdb..ab8fea9 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -84,6 +84,18 @@ class InterruptedPackageDict(TypedDict): state: str +class FilesystemUsageDict(TypedDict): + """How full one filesystem is, as ``df -Pk`` says, in bytes.""" + + #: Where it is mounted, e.g. "/" or "/boot". + mountpoint: str + total: int + used: int + #: What an ordinary user can still write: less than total minus used on a + #: filesystem that keeps blocks for root (ext4 keeps 5 %). + available: int + + class HostStatusDict(TypedDict): """What a host says about its own patch state (``HostStatusMixin.get_host_status``).""" @@ -103,6 +115,9 @@ class HostStatusDict(TypedDict): auto_updates_last_success: NotRequired[Optional[int]] #: How often they are meant to run, in seconds; 0 for every time the timer fires. auto_updates_interval: NotRequired[Optional[int]] + #: How full ``/``, ``/boot`` and ``/var`` are, each filesystem once; None when + #: it could not be read. + filesystems: NotRequired[Optional[List[FilesystemUsageDict]]] # --------------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 54c0d54..f8d3716 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.3.1" +version = "4.4.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_host_status.py b/tests/test_host_status.py index e9f3865..d646117 100644 --- a/tests/test_host_status.py +++ b/tests/test_host_status.py @@ -428,3 +428,81 @@ class TestBusyCheck: """Run over an exec channel, the command is the command line of the shell that runs it, and ``ps -o args`` lists it.""" assert not re.search(r"(^|[ /])unattended-upgrade( |$)", HOST_STATUS_COMMAND) + + +class TestFilesystems: + """How full the filesystems an upgrade writes to are: ``/``, ``/boot`` and + ``/var``, read with ``df -Pk`` and no root. + + pve-dual's root filesystem stood at 94 % with about 400 MB free for a week; + a kernel upgrade failed on it with "No space left on device" and left dpkg + halfway, and nothing had said the disk was full (NetOrk/netork#862). The + figures below are shaped after it (6.5 GiB, 400 MiB available): ext4 keeps + 5 % for root, so what is available is less than total minus used. + """ + + HEADER = "Filesystem 1024-blocks Used Available Capacity Mounted on" + PVE_ROOT = "/dev/mapper/pve-root 6815744 6065357 409600 94% /" + BOOT = "/dev/sda2 1046512 104860 941652 11% /boot" + + @staticmethod + def _report(*df_lines: str) -> str: + return _wire().replace("[timers]", "\n".join(["[filesystems]", *df_lines, "[timers]"])) + + def test_the_root_filesystem_in_bytes(self): + status = parse_host_status(self._report(self.HEADER, self.PVE_ROOT)) + + assert status["filesystems"] == [ + { + "mountpoint": "/", + "total": 6815744 * 1024, + "used": 6065357 * 1024, + "available": 409600 * 1024, + } + ] + + def test_a_separate_boot_partition(self): + status = parse_host_status(self._report(self.HEADER, self.PVE_ROOT, self.BOOT)) + + assert [fs["mountpoint"] for fs in status["filesystems"]] == ["/", "/boot"] + + def test_a_directory_on_root_is_root_once(self): + """``df /boot /var`` names the filesystem they live on, here ``/`` three times.""" + status = parse_host_status( + self._report(self.HEADER, self.PVE_ROOT, self.PVE_ROOT, self.PVE_ROOT) + ) + + assert [fs["mountpoint"] for fs in status["filesystems"]] == ["/"] + + def test_a_long_device_name_and_a_blank_in_the_mountpoint(self): + line = "/dev/disk/by-id/a very long name 1000 600 400 60% /mnt/my disk" + + (fs,) = parse_host_status(self._report(self.HEADER, line))["filesystems"] + + assert (fs["mountpoint"], fs["available"]) == ("/mnt/my disk", 400 * 1024) + + def test_a_filesystem_without_size_is_left_out(self): + """What a pseudo filesystem reports; it cannot run full.""" + status = parse_host_status(self._report(self.HEADER, "none 0 0 0 - /")) + + assert status["filesystems"] is None + + def test_without_the_section_it_is_unknown(self): + """An older command, or no df: None, not "nothing is full".""" + assert parse_host_status(_wire())["filesystems"] is None + + def test_df_that_printed_nothing_is_unknown(self): + assert parse_host_status(self._report())["filesystems"] is None + + def test_the_command_asks_for_the_three_without_root(self): + assert "df -Pk / /boot /var" in HOST_STATUS_COMMAND + assert "sudo" not in HOST_STATUS_COMMAND + + def test_the_command_reads_them_on_a_real_shell(self): + """POSIX df the way a host's sh runs it: ``/`` is always there, whatever it is.""" + out = subprocess.run( + ["sh", "-c", HOST_STATUS_COMMAND], capture_output=True, text=True, timeout=30 + ) + + mountpoints = [fs["mountpoint"] for fs in parse_host_status(out.stdout)["filesystems"]] + assert "/" in mountpoints -- 2.54.0