From deb233617c72a55d39f049b4c682b3961538205e Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Mon, 2 Nov 2026 08:42:48 +0100 Subject: [PATCH] fix(disks): raise on a failed lsblk, warn on failed smartctl calls get_disks swallowed a failed or garbled lsblk and returned [], the answer for a NAS without disks, so every disk vanished on a transient error. It now raises CommandErrorException; a failed /proc/mdstat read raises too instead of dropping every pool. A smartctl call that timed out, was refused by sudo or could not open the device was logged at debug level and read like a disk without SMART. It is now logged as a warning with the reason (exit status bits 0 and 1; the disk-state bits 2-7 are not a failed call). A disk whose verdict passed but that counts reallocated, pending or offline-uncorrectable sectors (ATA), grown defects (SAS), or sets a critical warning or counts media errors (NVMe) is reported as "warning", as the contract allows. get_disks shrinks below its baseline entries, which are tightened. Closes #16 --- CHANGELOG.md | 9 ++ README.md | 2 +- napalm_openmediavault/openmediavault.py | 134 ++++++++++++++----- tests/quality_baseline.json | 4 +- tests/test_disks.py | 166 ++++++++++++++++++++++-- 5 files changed, 271 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3fdda7..db84c28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,15 @@ All notable changes to this driver. The format follows - `get_disks()` reports a SCSI/SAS disk's temperature (`Current Drive Temperature`); it read `-1` for every SAS disk. A disk that prints `0 C` there, as QEMU's virtual disks do, still reads `-1`. +- `get_disks()` raises `CommandErrorException` when `lsblk` fails or prints no device + list, and lets a failed `/proc/mdstat` read raise; it returned `[]`, the answer for + a NAS without disks, so every disk vanished on a transient error (#16). +- `get_disks()` reports `warning` for a disk whose verdict passed but that counts lost + or unreadable sectors: ATA reallocated, pending or offline-uncorrectable sectors, + SAS grown defects, an NVMe critical-warning flag or NVMe media errors (#16). +- A failed `smartctl` call (timeout, sudo refused, device not opened) is logged at + warning level with the reason; it was logged at debug level and looked like a disk + without SMART. The disk still reads `unknown` (#16). ## [0.3.0] - 2026-10-23 diff --git a/README.md b/README.md index b9b24ef..09428b2 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ with driver("192.0.2.10", "admin", "secret", optional_args={"sudo_password": "se | Getter / action | | |---|---| | `get_facts` | napalm-linux's facts, `os_version` is `OMV ` of the `openmediavault` package | -| `get_disks` | `lsblk` for the disks, `smartctl -iA` for health, temperature and RPM | +| `get_disks` | `lsblk` for the disks, `smartctl -iHA` for health (`healthy`, `warning`, `failed`, `unknown`), temperature and RPM | | `get_disk_pools` | md arrays from `/proc/mdstat`, size and state from `mdadm --detail` | | `get_volumes` | mounted block-device filesystems from `df`, read-only flag from `/proc/mounts` | | `get_shares` | SMB, NFS and FTP shares from OMV's configuration database (`omv-confdbadm`) | diff --git a/napalm_openmediavault/openmediavault.py b/napalm_openmediavault/openmediavault.py index dc4b0d4..bbc6c4e 100644 --- a/napalm_openmediavault/openmediavault.py +++ b/napalm_openmediavault/openmediavault.py @@ -23,6 +23,7 @@ import re from shlex import quote as _shlex_quote from typing import Any +from napalm.base.exceptions import CommandErrorException from napalm_device_types import FingerprintRule, StorageDriver from napalm_device_types.models import ( DiskPoolDict, @@ -50,13 +51,29 @@ _HOSTNAME_MODULES = frozenset( _DIRTY_MODULES = "/var/lib/openmediavault/dirtymodules.json" +#: Every block device, as JSON; sizes in bytes. +_LSBLK = "lsblk -J -b -o NAME,SIZE,TYPE,ROTA,VENDOR,MODEL,SERIAL,TRAN 2>/dev/null" #: Identity, overall health verdict and attributes; ``-H`` alone prints the verdict. _SMARTCTL = "smartctl -iHA /dev/{name} 2>/dev/null" +_SMARTCTL_TIMEOUT = 15 +#: smartctl's exit status bits 0 and 1: the command line did not parse (sudo's own +#: refusal exits 1 as well) or the device could not be opened. Bits 2-7 describe the +#: disk, not the call. +_SMARTCTL_CALL_FAILED = 0b11 #: The overall verdict of an ATA or NVMe disk: ``PASSED`` or ``FAILED!``. _SMART_VERDICT_RE = re.compile(r"^SMART overall-health self-assessment test result:\s*(\S+)", re.M) #: A SCSI/SAS disk's: ``OK``, or the failure it predicts. _SMART_SCSI_STATUS_RE = re.compile(r"^SMART Health Status:\s*(.+?)\s*$", re.M) _SMART_VERDICTS = {"PASSED": "healthy", "FAILED": "failed"} +#: ATA attributes counting sectors the disk lost or could not read, and the raw value +#: of each (the column after WHEN_FAILED). +_SMART_ATA_DEGRADED = ("Reallocated_Sector_Ct", "Current_Pending_Sector", "Offline_Uncorrectable") +_SMART_ATA_RAW_RE = re.compile(r"^\s*\d+\s+(\S+)\s+0x[0-9a-fA-F]+\s+(?:\S+\s+){6}(\d+)", re.M) +#: An NVMe disk's own warning flags, and the media errors it counted. +_SMART_NVME_CRITICAL_RE = re.compile(r"^Critical Warning:\s+0x([0-9a-fA-F]+)\s*$", re.M) +_SMART_NVME_MEDIA_ERRORS_RE = re.compile(r"^Media and Data Integrity Errors:\s+([\d,]+)\s*$", re.M) +#: A SCSI/SAS disk's reassigned blocks. +_SMART_SCSI_DEFECTS_RE = re.compile(r"^Elements in grown defect list:\s+(\d+)\s*$", re.M) #: An ATA disk's temperature attribute, an NVMe disk's ``Temperature:`` line. _SMART_TEMP_RE = re.compile( r"(?:Temperature_Celsius|Current Temperature|Temperature:)[^\n]*?(\d+)\s*(?:Celsius|\(|$)" @@ -69,11 +86,22 @@ _MDSTAT_MEMBER_RE = re.compile(r"(\S+?)\[\d+\]") def smart_health(output: str) -> str: - """The overall SMART verdict in ``smartctl -H`` output. + """A disk's health from ``smartctl -iHA`` output. - ``"healthy"`` or ``"failed"``; ``"unknown"`` when smartctl printed none (no - SMART support, a USB bridge without pass-through). + ``"failed"`` when the overall verdict says so; ``"warning"`` when it passed but + the disk counts lost or unreadable sectors (ATA reallocated, pending or offline + uncorrectable sectors, SAS grown defects) or, for NVMe, sets a critical warning + flag or counts media errors; ``"healthy"`` otherwise; ``"unknown"`` when + smartctl printed no verdict (no SMART support, a USB bridge without + pass-through). """ + verdict = _smart_verdict(output) + if verdict == "healthy" and _smart_degraded(output): + return "warning" + return verdict + + +def _smart_verdict(output: str) -> str: verdict = _SMART_VERDICT_RE.search(output) if verdict: return _SMART_VERDICTS.get(verdict.group(1).rstrip("!"), "unknown") @@ -83,6 +111,22 @@ def smart_health(output: str) -> str: return "unknown" +def _smart_degraded(output: str) -> bool: + ata = any( + name in _SMART_ATA_DEGRADED and int(raw) > 0 + for name, raw in _SMART_ATA_RAW_RE.findall(output) + ) + critical = _SMART_NVME_CRITICAL_RE.search(output) + media = _SMART_NVME_MEDIA_ERRORS_RE.search(output) + defects = _SMART_SCSI_DEFECTS_RE.search(output) + return ( + ata + or bool(critical and int(critical.group(1), 16)) + or bool(media and int(media.group(1).replace(",", ""))) + or bool(defects and int(defects.group(1))) + ) + + def smart_temperature(output: str) -> int: """A disk's temperature in Celsius from ``smartctl -A`` output; ``-1`` if none.""" found = _SMART_TEMP_RE.search(output) @@ -236,7 +280,9 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver): * size (int) - raw capacity in bytes * rpm (int) - rotational speed; ``0`` for SSD/NVMe * temperature (int) - temperature in Celsius; ``-1`` if unavailable - * health (string) - ``"healthy"``, ``"failed"``, or ``"unknown"`` + * health (string) - ``"healthy"``, ``"warning"``, ``"failed"``, or + ``"unknown"`` (see :func:`smart_health`); a smartctl call that failed is + logged as a warning and leaves the disk ``"unknown"`` * pool (string) - md array name if member; empty string otherwise Example:: @@ -255,19 +301,13 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver): "pool": "md1", }, ] + + :raises CommandErrorException: when lsblk printed no device list. """ result: list[PhysicalDiskDict] = [] - - try: - raw = self._send( - "lsblk -J -b -o NAME,SIZE,TYPE,ROTA,VENDOR,MODEL,SERIAL,TRAN 2>/dev/null" - ) - lsblk = _json.loads(raw) - except Exception: - return result - + devices = self._lsblk() members = self._md_members() - for dev in lsblk.get("blockdevices", []): + for dev in devices: if dev.get("type") != "disk": continue name = dev.get("name", "") @@ -307,28 +347,56 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver): return result - def _smart(self, name: str, disk_type: str) -> tuple[str, int, int]: - """A disk's SMART health, temperature (``-1`` if unknown) and rpm (``0``).""" - health, temperature, rpm = "unknown", -1, 0 + def _lsblk(self) -> list[dict[str, Any]]: + """The block devices lsblk lists; a failed or garbled read raises.""" + raw = self._send(_LSBLK) try: - smart = self._sudo(_SMARTCTL.format(name=name), read_timeout=15) - health = smart_health(smart) - temperature = smart_temperature(smart) - if disk_type == "hdd": - rpm_m = re.search(r"Rotation Rate:\s+(\d+)\s+rpm", smart) - if rpm_m: - rpm = int(rpm_m.group(1)) - except Exception: - logger.debug("%s: smartctl for %s: ignored", self.hostname, name, exc_info=True) - return health, temperature, rpm + devices = _json.loads(raw)["blockdevices"] + except (ValueError, KeyError, TypeError) as exc: + raise CommandErrorException( + f"lsblk on {self.hostname} printed no device list: {raw[-300:]!r}" + ) from exc + if not isinstance(devices, list): + raise CommandErrorException(f"lsblk on {self.hostname}: blockdevices is no list") + return devices + + def _smart(self, name: str, disk_type: str) -> tuple[str, int, int]: + """A disk's SMART health, temperature (``-1`` if unknown) and rpm (``0``). + + A smartctl call that failed (timeout, sudo refused, device not opened) is + logged as a warning, so it is told apart from a disk without SMART; the + disk stays in the inventory as ``"unknown"``. + """ + try: + smart, status = self._sudo_status( + _SMARTCTL.format(name=name), read_timeout=_SMARTCTL_TIMEOUT + ) + except OSError as exc: # TimeoutError and the socket's own errors + logger.warning("%s: smartctl for %s failed: %s", self.hostname, name, exc) + return "unknown", -1, 0 + if status is not None and status & _SMARTCTL_CALL_FAILED: + logger.warning( + "%s: smartctl for %s failed (exit status %s): %s", + self.hostname, + name, + status, + smart[-300:], + ) + return "unknown", -1, 0 + rpm = 0 + if disk_type == "hdd": + rpm_m = re.search(r"Rotation Rate:\s+(\d+)\s+rpm", smart) + if rpm_m: + rpm = int(rpm_m.group(1)) + return smart_health(smart), smart_temperature(smart), rpm def _md_members(self) -> dict[str, str]: - """The md array of each member device, from one read of ``/proc/mdstat``.""" - try: - return md_members(self._send("cat /proc/mdstat 2>/dev/null")) - except Exception: - logger.debug("%s: /proc/mdstat not read: ignored", self.hostname, exc_info=True) - return {} + """The md array of each member device, from one read of ``/proc/mdstat``. + + A host without the md driver has no ``/proc/mdstat`` and no members; a + failed read raises. + """ + return md_members(self._send("cat /proc/mdstat 2>/dev/null")) # ── md RAID pools ────────────────────────────────────────────────────────── diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 6af16f8..2ec0fd9 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -1,11 +1,11 @@ { "coverage_floor": 15.0, "function_complexity": { - "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 12 + "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 11 }, "function_lines": { "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disk_pools": 81, - "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 87, + "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 83, "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_storage_services": 58, "napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_volumes": 90 }, diff --git a/tests/test_disks.py b/tests/test_disks.py index 2c43b40..8cfc7f4 100644 --- a/tests/test_disks.py +++ b/tests/test_disks.py @@ -1,4 +1,4 @@ -"""get_disks: the overall SMART verdict, and the md array each disk belongs to. +"""get_disks: the SMART health, the md array each disk belongs to, and failed reads. smartctl prints the overall verdict only for ``-H``; ``-i`` (identity) and ``-A`` (attributes) never do. The fake host below answers smartctl with a recorded @@ -13,10 +13,13 @@ numbers, WWNs and LU ids replaced. from __future__ import annotations import json +import logging import re from pathlib import Path +from typing import Any import pytest +from napalm.base.exceptions import CommandErrorException from napalm_openmediavault.openmediavault import OpenMediaVaultDriver, smart_health @@ -52,6 +55,39 @@ SAS_PREDICTED = _replaced( "SMART Health Status: OK\n", "SMART Health Status: FIRMWARE IMPENDING FAILURE TOO MANY BLOCK REASSIGNS [asc=5d, ascq=64]\n", ) +# No degraded disk was at hand either: these put a count in place of a recorded zero. +_ATA_REALLOCATED = ( + " 5 Reallocated_Sector_Ct 0x0000 100 100 000 Old_age Offline - 0\n" +) +ATA_REALLOCATED = _replaced( + ATA_PASSED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8") +) +_ATA_PENDING = ( + "197 Current_Pending_Sector 0x0000 100 100 000 Old_age Offline - 0\n" +) +ATA_PENDING = _replaced(ATA_PASSED, _ATA_PENDING, _ATA_PENDING.replace("- 0", "- 2")) +_ATA_OFFLINE_UNC = ( + "198 Offline_Uncorrectable 0x0000 100 100 000 Old_age Offline - 0\n" +) +ATA_OFFLINE_UNC = _replaced( + ATA_PASSED, _ATA_OFFLINE_UNC, _ATA_OFFLINE_UNC.replace("- 0", "- 1") +) +ATA_FAILED_REALLOCATED = _replaced( + ATA_FAILED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8") +) +NVME_CRITICAL_WARNING = _replaced( + NVME_PASSED, + "Critical Warning: 0x00\n", + "Critical Warning: 0x04\n", +) +NVME_MEDIA_ERRORS = _replaced( + NVME_PASSED, + "Media and Data Integrity Errors: 0\n", + "Media and Data Integrity Errors: 3\n", +) +SAS_GROWN_DEFECTS = _replaced( + SAS_OK, "Elements in grown defect list: 0\n", "Elements in grown defect list: 12\n" +) #: What ``smartctl -iA`` prints: no verdict, but the attribute table's WHEN_FAILED header. ATA_WITHOUT_VERDICT = _replaced(ATA_PASSED, _ATA_VERDICT, "") @@ -66,16 +102,29 @@ def smartctl(flags: str, recording: str) -> str: class Nas: - """The NAS as ``_send`` and ``_sudo`` see it: lsblk, /proc/mdstat and smartctl.""" + """The NAS as ``_send`` and ``_sudo_status`` see it: lsblk, /proc/mdstat, smartctl. - def __init__(self, disks: dict[str, tuple[str, str]], mdstat: str = "") -> None: + A disk's smartctl answer is its recorded output (exit status 0), an + ``(output, exit status)`` pair, or an exception the transport raises. *lsblk* and + *mdstat* given as an exception are raised the same way. + """ + + def __init__( + self, + disks: dict[str, tuple[str, Any]], + mdstat: str | Exception = "", + lsblk: str | None = None, + ) -> None: #: name -> (tran, recorded ``smartctl -iHA`` output) self.disks = disks self.mdstat = mdstat + self.lsblk = lsblk self.commands: list[str] = [] def send(self, command: str, read_timeout: float = 100) -> str: self.commands.append(command) + if command.startswith("lsblk ") and self.lsblk is not None: + return self.lsblk if command.startswith("lsblk "): return json.dumps( { @@ -84,7 +133,7 @@ class Nas: "name": name, "size": 6001175126016, "type": "disk", - "rota": tran != "nvme" and "Solid State Device" not in recording, + "rota": tran != "nvme" and "Solid State Device" not in str(recording), "vendor": "", "model": "m", "serial": "s", @@ -95,21 +144,26 @@ class Nas: } ) if command.startswith("cat /proc/mdstat"): + if isinstance(self.mdstat, Exception): + raise self.mdstat return self.mdstat raise AssertionError(command) - def sudo(self, command: str, read_timeout: float = 100) -> str: + def sudo_status(self, command: str, read_timeout: float = 100) -> tuple[str, int | None]: self.commands.append(command) m = re.match(r"smartctl -(\w+) /dev/(\w+)", command) assert m, command - _tran, recording = self.disks[m.group(2)] - return smartctl(m.group(1), recording) + _tran, answer = self.disks[m.group(2)] + if isinstance(answer, Exception): + raise answer + output, status = answer if isinstance(answer, tuple) else (answer, 0) + return smartctl(m.group(1), output), status def _driver(nas: Nas) -> OpenMediaVaultDriver: driver = OpenMediaVaultDriver("nas", "admin", "p") driver._send = nas.send # type: ignore[method-assign] - driver._sudo = nas.sudo # type: ignore[method-assign] + driver._sudo_status = nas.sudo_status # type: ignore[method-assign] return driver @@ -173,6 +227,15 @@ def test_a_disk_without_smart_has_no_temperature(): ("", "unknown"), # The attribute table's WHEN_FAILED column is no verdict. (ATA_WITHOUT_VERDICT, "unknown"), + # A passing disk that is losing sectors is degraded (#16). + (ATA_REALLOCATED, "warning"), + (ATA_PENDING, "warning"), + (ATA_OFFLINE_UNC, "warning"), + (NVME_CRITICAL_WARNING, "warning"), + (NVME_MEDIA_ERRORS, "warning"), + (SAS_GROWN_DEFECTS, "warning"), + # The verdict outranks the counters. + (ATA_FAILED_REALLOCATED, "failed"), ], ids=[ "ata-passed", @@ -183,12 +246,99 @@ def test_a_disk_without_smart_has_no_temperature(): "no-smart", "empty", "when-failed-column", + "ata-reallocated", + "ata-pending", + "ata-offline-uncorrectable", + "nvme-critical-warning", + "nvme-media-errors", + "sas-grown-defects", + "ata-failed-and-reallocated", ], ) def test_smart_health_reads_the_verdict_line_only(output, health): assert smart_health(output) == health +def test_a_degraded_disk_is_reported_as_warning(): + nas = Nas({"sda": ("sata", ATA_REALLOCATED)}) + + assert _by_slot(nas)["/dev/sda"]["health"] == "warning" + + +# ── failed reads (#16) ────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "lsblk", + [ + "", + "sudo: a terminal is required to read the password", + "lsblk: unknown column: TRAN", + '["not", "an", "object"]', + ], + ids=["nothing", "sudo", "lsblk-error", "not-an-object"], +) +def test_a_failed_lsblk_raises(lsblk): + """``[]`` is a NAS without disks; a failed read must not look like one.""" + nas = Nas({}, lsblk=lsblk) + + with pytest.raises(CommandErrorException, match="lsblk"): + _driver(nas).get_disks() + + +def test_a_nas_without_disks_has_none(): + nas = Nas({}, lsblk='{"blockdevices": []}') + + assert _driver(nas).get_disks() == [] + + +def test_a_smartctl_timeout_is_logged_as_a_warning(caplog): + nas = Nas( + { + "sda": ("sata", TimeoutError("'smartctl -iHA /dev/sda' did not finish within 15s")), + "sdb": ("sata", ATA_PASSED), + } + ) + + with caplog.at_level(logging.WARNING): + disks = _by_slot(nas) + + assert (disks["/dev/sda"]["health"], disks["/dev/sda"]["temperature"]) == ("unknown", -1) + assert disks["/dev/sdb"]["health"] == "healthy" + warned = [r for r in caplog.records if r.levelno == logging.WARNING] + assert any("sda" in r.getMessage() and "smartctl" in r.getMessage() for r in warned) + + +def test_a_refused_sudo_for_smartctl_is_logged_as_a_warning(caplog): + # sudo's own refusal exits 1, which smartctl uses for "command line did not parse". + nas = Nas({"sda": ("sata", ("sudo: a password is required", 1))}) + + with caplog.at_level(logging.WARNING): + disk = _by_slot(nas)["/dev/sda"] + + assert disk["health"] == "unknown" + warned = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("sda" in m and "sudo: a password is required" in m for m in warned) + + +def test_a_disk_status_in_smartctls_exit_code_is_no_failed_call(caplog): + # Bits 3-7 of smartctl's exit status describe the disk, not the call. + nas = Nas({"sda": ("sata", (ATA_FAILED, 8))}) + + with caplog.at_level(logging.WARNING): + disk = _by_slot(nas)["/dev/sda"] + + assert disk["health"] == "failed" + assert not [r for r in caplog.records if r.levelno >= logging.WARNING] + + +def test_a_failed_mdstat_read_raises(): + nas = Nas({"sda": ("sata", ATA_PASSED)}, mdstat=TimeoutError("no data within 100s")) + + with pytest.raises(TimeoutError): + _driver(nas).get_disks() + + # ── md membership ─────────────────────────────────────────────────────────────── # The recorded hosts have no md array (MDSTAT_NO_ARRAYS), so the arrays below are -- 2.54.1