feat: report how full /, /boot and /var are in the host status

A Proxmox host's root filesystem stood at 94 % with about 400 MB free for a
week; a kernel upgrade failed on it with "No space left on device" and left
dpkg halfway, and nothing had said the disk was full (NetOrk/netork#862).

HOST_STATUS_COMMAND now also runs POSIX `df -Pk / /boot /var`, without root.
get_host_status() returns `filesystems`: each filesystem once (a directory
that is not a mount of its own is the filesystem it lives on), with its
mountpoint, total, used and available bytes, or None when df said nothing.
`available` is what df says an ordinary user can still write, which on ext4
is less than total minus used. How full is too full is the caller's decision.

Version 4.4.0.

README.md

@@ -84,7 +84,8 @@ a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`,
 kernel of the running flavour installed), whether it patches itself (unattended-upgrades,
 dnf-automatic) and when that last succeeded (apt's `upgrade-stamp`), and whether dpkg was
 left halfway (a journal in `/var/lib/dpkg/updates` or a half-configured package; unknown
-while a package manager is running). `package_updates` holds the shared apt and dnf parsers: apt's suites become
+while a package manager is running), and how full `/`, `/boot` and `/var` are (`df -Pk`,
+each filesystem once, in bytes; how full is too full is the caller's decision). `package_updates` holds the shared apt and dnf parsers: apt's suites become
 an update's `origin`, a `-security` suite makes it a security update, and dnf's security
 advisories do the same.
 

napalm_device_types/host_status.py

@@ -41,6 +41,15 @@ unattended-upgrade is running, packages are halfway on purpose -- also between
 two of apt's dpkg calls -- so the answer is then ``None``, as on a host without
 dpkg. unattended-upgrade is told from Ubuntu's idle shutdown helper by its
 command line (:data:`BUSY_CHECK`).
+
+**Filesystems** are how full ``/``, ``/boot`` and ``/var`` are, from POSIX
+``df -Pk``: where an upgrade unpacks a kernel, builds its initrd and keeps the
+package files. A kernel upgrade that ran out of room there left dpkg halfway on
+a Proxmox host whose root had been 94 % full for a week (NetOrk/netork#862). A
+directory that is not a mount of its own is reported as the filesystem it lives
+on, so each filesystem appears once. ``available`` is what df says an ordinary
+user can still write, which on ext4 is less than total minus used. How full is
+too full is not decided here.
 """
 
 from __future__ import annotations
@@ -48,7 +57,11 @@ from __future__ import annotations
 import re
 from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 
-from napalm_device_types.models import HostStatusDict, InterruptedPackageDict
+from napalm_device_types.models import (
+    FilesystemUsageDict,
+    HostStatusDict,
+    InterruptedPackageDict,
+)
 from napalm_device_types.terminal import strip_terminal_codes
 
 _BEGIN = "HSTAT_BEGIN"
@@ -58,6 +71,8 @@ _APT_TIMER = "apt-daily-upgrade.timer"
 _DNF_TIMERS = ("dnf-automatic.timer", "dnf-automatic-install.timer")
 _UPGRADE_STAMP = "/var/lib/apt/periodic/upgrade-stamp"
 _DPKG_JOURNAL = "/var/lib/dpkg/updates"
+#: Where an upgrade writes: the kernel's modules, its initrd, the package files.
+_FILESYSTEMS = "/ /boot /var"
 #: Prints ``[dpkg-busy]`` while a process that leaves packages halfway is working:
 #: dpkg, apt, apt-get or aptitude by name, unattended-upgrade by its command line.
 #: Its name is cut to 15 characters, ``unattended-upgr``, the same as that of
@@ -91,6 +106,7 @@ HOST_STATUS_COMMAND = (
     f"echo '[dpkg-journal]'; ls -1 {_DPKG_JOURNAL} 2>/dev/null | head -n 20; "
     "echo '[dpkg-audit]'; dpkg-query -W -f='${db:Status-Abbrev} ${Package}\\n' 2>/dev/null "
     "| awk 'substr($0, 2, 1) ~ /[HUFWt]/ || substr($0, 3, 1) == \"R\"' | head -n 50; fi; "
+    f"echo '[filesystems]'; df -Pk {_FILESYSTEMS}; "
     f"echo '[timers]'; for u in {_APT_TIMER} {' '.join(_DNF_TIMERS)}; do "
     'printf \'%s %s\\n\' "$u" "$(systemctl is-enabled "$u" 2>/dev/null)"; done; '
     "printf '%s%s\\n' HSTAT_ END; } 2>/dev/null | cat"
@@ -101,6 +117,9 @@ _OFF_STATES = frozenset({"disabled", "masked"})
 _INTERVAL = re.compile(r"(\d+)([smhd]?)")
 _UNIT_SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "": 86400}
 _JOURNAL_ENTRY = re.compile(r"\d+")
+#: One line of ``df -P``: device, 1024-blocks, used, available, capacity, mountpoint.
+#: The device and the mountpoint may hold blanks; the four numbers between them do not.
+_DF_LINE = re.compile(r"^.+?\s+(\d+)\s+(\d+)\s+(\d+)\s+(?:\d+%|-)\s+(/.*)$")
 #: dpkg's second status letter -> its word, for the states a cut-off run leaves.
 _HALFWAY = {
     "H": "half-installed",
@@ -236,6 +255,22 @@ def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[Interrup
     return bool(packages) or journal, packages
 
 
+def _filesystems(sections: Dict[str, List[str]]) -> Optional[List[FilesystemUsageDict]]:
+    """Each filesystem df named, once and in its order; None when it named none."""
+    found: Dict[str, FilesystemUsageDict] = {}
+    for line in sections.get("filesystems") or []:
+        match = _DF_LINE.match(line)
+        if not match or int(match.group(1)) == 0:
+            continue
+        total, used, available = (int(match.group(i)) * 1024 for i in (1, 2, 3))
+        mountpoint = match.group(4)
+        found.setdefault(
+            mountpoint,
+            {"mountpoint": mountpoint, "total": total, "used": used, "available": available},
+        )
+    return list(found.values()) or None
+
+
 def parse_host_status(output: str) -> HostStatusDict:
     """Parse what :data:`HOST_STATUS_COMMAND` printed.
 
@@ -252,6 +287,7 @@ def parse_host_status(output: str) -> HostStatusDict:
         "auto_updates_interval": _interval(sections),
         "package_manager_interrupted": interrupted,
         "interrupted_packages": packages,
+        "filesystems": _filesystems(sections),
     }
 
 
@@ -282,6 +318,7 @@ class HostStatusMixin:
         * auto_updates_interval (seconds or None)
         * package_manager_interrupted (bool or None)
         * interrupted_packages (list of {name, state})
+        * filesystems (list of {mountpoint, total, used, available} in bytes, or None)
 
         :raises ValueError: if the host's output carried no intact report.
         """

napalm_device_types/models.py

@@ -84,6 +84,18 @@ class InterruptedPackageDict(TypedDict):
     state: str
 
 
+class FilesystemUsageDict(TypedDict):
+    """How full one filesystem is, as ``df -Pk`` says, in bytes."""
+
+    #: Where it is mounted, e.g. "/" or "/boot".
+    mountpoint: str
+    total: int
+    used: int
+    #: What an ordinary user can still write: less than total minus used on a
+    #: filesystem that keeps blocks for root (ext4 keeps 5 %).
+    available: int
+
+
 class HostStatusDict(TypedDict):
     """What a host says about its own patch state (``HostStatusMixin.get_host_status``)."""
 
@@ -103,6 +115,9 @@ class HostStatusDict(TypedDict):
     auto_updates_last_success: NotRequired[Optional[int]]
     #: How often they are meant to run, in seconds; 0 for every time the timer fires.
     auto_updates_interval: NotRequired[Optional[int]]
+    #: How full ``/``, ``/boot`` and ``/var`` are, each filesystem once; None when
+    #: it could not be read.
+    filesystems: NotRequired[Optional[List[FilesystemUsageDict]]]
 
 
 # ---------------------------------------------------------------------------

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "napalm-device-types"
-version = "4.3.1"
+version = "4.4.0"
 description = "Abstract device-type base classes for NAPALM drivers"
 readme = "README.md"
 requires-python = ">=3.10"

tests/test_host_status.py

@@ -428,3 +428,81 @@ class TestBusyCheck:
         """Run over an exec channel, the command is the command line of the shell
         that runs it, and ``ps -o args`` lists it."""
         assert not re.search(r"(^|[ /])unattended-upgrade( |$)", HOST_STATUS_COMMAND)
+
+
+class TestFilesystems:
+    """How full the filesystems an upgrade writes to are: ``/``, ``/boot`` and
+    ``/var``, read with ``df -Pk`` and no root.
+
+    pve-dual's root filesystem stood at 94 % with about 400 MB free for a week;
+    a kernel upgrade failed on it with "No space left on device" and left dpkg
+    halfway, and nothing had said the disk was full (NetOrk/netork#862). The
+    figures below are shaped after it (6.5 GiB, 400 MiB available): ext4 keeps
+    5 % for root, so what is available is less than total minus used.
+    """
+
+    HEADER = "Filesystem 1024-blocks Used Available Capacity Mounted on"
+    PVE_ROOT = "/dev/mapper/pve-root 6815744 6065357 409600 94% /"
+    BOOT = "/dev/sda2 1046512 104860 941652 11% /boot"
+
+    @staticmethod
+    def _report(*df_lines: str) -> str:
+        return _wire().replace("[timers]", "\n".join(["[filesystems]", *df_lines, "[timers]"]))
+
+    def test_the_root_filesystem_in_bytes(self):
+        status = parse_host_status(self._report(self.HEADER, self.PVE_ROOT))
+
+        assert status["filesystems"] == [
+            {
+                "mountpoint": "/",
+                "total": 6815744 * 1024,
+                "used": 6065357 * 1024,
+                "available": 409600 * 1024,
+            }
+        ]
+
+    def test_a_separate_boot_partition(self):
+        status = parse_host_status(self._report(self.HEADER, self.PVE_ROOT, self.BOOT))
+
+        assert [fs["mountpoint"] for fs in status["filesystems"]] == ["/", "/boot"]
+
+    def test_a_directory_on_root_is_root_once(self):
+        """``df /boot /var`` names the filesystem they live on, here ``/`` three times."""
+        status = parse_host_status(
+            self._report(self.HEADER, self.PVE_ROOT, self.PVE_ROOT, self.PVE_ROOT)
+        )
+
+        assert [fs["mountpoint"] for fs in status["filesystems"]] == ["/"]
+
+    def test_a_long_device_name_and_a_blank_in_the_mountpoint(self):
+        line = "/dev/disk/by-id/a very long name 1000 600 400 60% /mnt/my disk"
+
+        (fs,) = parse_host_status(self._report(self.HEADER, line))["filesystems"]
+
+        assert (fs["mountpoint"], fs["available"]) == ("/mnt/my disk", 400 * 1024)
+
+    def test_a_filesystem_without_size_is_left_out(self):
+        """What a pseudo filesystem reports; it cannot run full."""
+        status = parse_host_status(self._report(self.HEADER, "none 0 0 0 - /"))
+
+        assert status["filesystems"] is None
+
+    def test_without_the_section_it_is_unknown(self):
+        """An older command, or no df: None, not "nothing is full"."""
+        assert parse_host_status(_wire())["filesystems"] is None
+
+    def test_df_that_printed_nothing_is_unknown(self):
+        assert parse_host_status(self._report())["filesystems"] is None
+
+    def test_the_command_asks_for_the_three_without_root(self):
+        assert "df -Pk / /boot /var" in HOST_STATUS_COMMAND
+        assert "sudo" not in HOST_STATUS_COMMAND
+
+    def test_the_command_reads_them_on_a_real_shell(self):
+        """POSIX df the way a host's sh runs it: ``/`` is always there, whatever it is."""
+        out = subprocess.run(
+            ["sh", "-c", HOST_STATUS_COMMAND], capture_output=True, text=True, timeout=30
+        )
+
+        mountpoints = [fs["mountpoint"] for fs in parse_host_status(out.stdout)["filesystems"]]
+        assert "/" in mountpoints