Merge pull request 'fix(disks): raise on a failed lsblk, warn on failed smartctl calls (#16)' (#17) from fix/bugs-2026-11-02 into main
driver / standard (push) Successful in 18s
driver / lint (push) Successful in 14s
driver / types (push) Successful in 1m12s
driver / test (3.13) (push) Successful in 1m4s
driver / test (3.12) (push) Successful in 1m14s
driver / lowest (push) Successful in 35s
driver / build (push) Successful in 32s
driver / security (push) Successful in 1m26s
CI / driver (push) Successful in 3m4s
driver / publish (push) Skipped

This commit was merged in pull request #17.
This commit is contained in:
christianmanivong committed 2026-11-02 08:23:46 +00:00
commit a0d28fc939
5 files changed
+271 -44

No files matched your search

+9
View File
@@ -46,6 +46,15 @@ All notable changes to this driver. The format follows
- `get_disks()` reports a SCSI/SAS disk's temperature (`Current Drive Temperature`);
it read `-1` for every SAS disk. A disk that prints `0 C` there, as QEMU's virtual
disks do, still reads `-1`.
- `get_disks()` raises `CommandErrorException` when `lsblk` fails or prints no device
list, and lets a failed `/proc/mdstat` read raise; it returned `[]`, the answer for
a NAS without disks, so every disk vanished on a transient error (#16).
- `get_disks()` reports `warning` for a disk whose verdict passed but that counts lost
or unreadable sectors: ATA reallocated, pending or offline-uncorrectable sectors,
SAS grown defects, an NVMe critical-warning flag or NVMe media errors (#16).
- A failed `smartctl` call (timeout, sudo refused, device not opened) is logged at
warning level with the reason; it was logged at debug level and looked like a disk
without SMART. The disk still reads `unknown` (#16).
## [0.3.0] - 2026-10-23
+1 -1
View File
@@ -35,7 +35,7 @@ with driver("192.0.2.10", "admin", "secret", optional_args={"sudo_password": "se
| Getter / action | |
|---|---|
| `get_facts` | napalm-linux's facts, `os_version` is `OMV <version>` of the `openmediavault` package |
| `get_disks` | `lsblk` for the disks, `smartctl -iA` for health, temperature and RPM |
| `get_disks` | `lsblk` for the disks, `smartctl -iHA` for health (`healthy`, `warning`, `failed`, `unknown`), temperature and RPM |
| `get_disk_pools` | md arrays from `/proc/mdstat`, size and state from `mdadm --detail` |
| `get_volumes` | mounted block-device filesystems from `df`, read-only flag from `/proc/mounts` |
| `get_shares` | SMB, NFS and FTP shares from OMV's configuration database (`omv-confdbadm`) |
+101 -33
View File
@@ -23,6 +23,7 @@ import re
from shlex import quote as _shlex_quote
from typing import Any
from napalm.base.exceptions import CommandErrorException
from napalm_device_types import FingerprintRule, StorageDriver
from napalm_device_types.models import (
DiskPoolDict,
@@ -50,13 +51,29 @@ _HOSTNAME_MODULES = frozenset(
_DIRTY_MODULES = "/var/lib/openmediavault/dirtymodules.json"
#: Every block device, as JSON; sizes in bytes.
_LSBLK = "lsblk -J -b -o NAME,SIZE,TYPE,ROTA,VENDOR,MODEL,SERIAL,TRAN 2>/dev/null"
#: Identity, overall health verdict and attributes; ``-H`` alone prints the verdict.
_SMARTCTL = "smartctl -iHA /dev/{name} 2>/dev/null"
_SMARTCTL_TIMEOUT = 15
#: smartctl's exit status bits 0 and 1: the command line did not parse (sudo's own
#: refusal exits 1 as well) or the device could not be opened. Bits 2-7 describe the
#: disk, not the call.
_SMARTCTL_CALL_FAILED = 0b11
#: The overall verdict of an ATA or NVMe disk: ``PASSED`` or ``FAILED!``.
_SMART_VERDICT_RE = re.compile(r"^SMART overall-health self-assessment test result:\s*(\S+)", re.M)
#: A SCSI/SAS disk's: ``OK``, or the failure it predicts.
_SMART_SCSI_STATUS_RE = re.compile(r"^SMART Health Status:\s*(.+?)\s*$", re.M)
_SMART_VERDICTS = {"PASSED": "healthy", "FAILED": "failed"}
#: ATA attributes counting sectors the disk lost or could not read, and the raw value
#: of each (the column after WHEN_FAILED).
_SMART_ATA_DEGRADED = ("Reallocated_Sector_Ct", "Current_Pending_Sector", "Offline_Uncorrectable")
_SMART_ATA_RAW_RE = re.compile(r"^\s*\d+\s+(\S+)\s+0x[0-9a-fA-F]+\s+(?:\S+\s+){6}(\d+)", re.M)
#: An NVMe disk's own warning flags, and the media errors it counted.
_SMART_NVME_CRITICAL_RE = re.compile(r"^Critical Warning:\s+0x([0-9a-fA-F]+)\s*$", re.M)
_SMART_NVME_MEDIA_ERRORS_RE = re.compile(r"^Media and Data Integrity Errors:\s+([\d,]+)\s*$", re.M)
#: A SCSI/SAS disk's reassigned blocks.
_SMART_SCSI_DEFECTS_RE = re.compile(r"^Elements in grown defect list:\s+(\d+)\s*$", re.M)
#: An ATA disk's temperature attribute, an NVMe disk's ``Temperature:`` line.
_SMART_TEMP_RE = re.compile(
r"(?:Temperature_Celsius|Current Temperature|Temperature:)[^\n]*?(\d+)\s*(?:Celsius|\(|$)"
@@ -69,11 +86,22 @@ _MDSTAT_MEMBER_RE = re.compile(r"(\S+?)\[\d+\]")
def smart_health(output: str) -> str:
"""The overall SMART verdict in ``smartctl -H`` output.
"""A disk's health from ``smartctl -iHA`` output.
``"healthy"`` or ``"failed"``; ``"unknown"`` when smartctl printed none (no
SMART support, a USB bridge without pass-through).
``"failed"`` when the overall verdict says so; ``"warning"`` when it passed but
the disk counts lost or unreadable sectors (ATA reallocated, pending or offline
uncorrectable sectors, SAS grown defects) or, for NVMe, sets a critical warning
flag or counts media errors; ``"healthy"`` otherwise; ``"unknown"`` when
smartctl printed no verdict (no SMART support, a USB bridge without
pass-through).
"""
verdict = _smart_verdict(output)
if verdict == "healthy" and _smart_degraded(output):
return "warning"
return verdict
def _smart_verdict(output: str) -> str:
verdict = _SMART_VERDICT_RE.search(output)
if verdict:
return _SMART_VERDICTS.get(verdict.group(1).rstrip("!"), "unknown")
@@ -83,6 +111,22 @@ def smart_health(output: str) -> str:
return "unknown"
def _smart_degraded(output: str) -> bool:
ata = any(
name in _SMART_ATA_DEGRADED and int(raw) > 0
for name, raw in _SMART_ATA_RAW_RE.findall(output)
)
critical = _SMART_NVME_CRITICAL_RE.search(output)
media = _SMART_NVME_MEDIA_ERRORS_RE.search(output)
defects = _SMART_SCSI_DEFECTS_RE.search(output)
return (
ata
or bool(critical and int(critical.group(1), 16))
or bool(media and int(media.group(1).replace(",", "")))
or bool(defects and int(defects.group(1)))
)
def smart_temperature(output: str) -> int:
"""A disk's temperature in Celsius from ``smartctl -A`` output; ``-1`` if none."""
found = _SMART_TEMP_RE.search(output)
@@ -236,7 +280,9 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver):
* size (int) - raw capacity in bytes
* rpm (int) - rotational speed; ``0`` for SSD/NVMe
* temperature (int) - temperature in Celsius; ``-1`` if unavailable
* health (string) - ``"healthy"``, ``"failed"``, or ``"unknown"``
* health (string) - ``"healthy"``, ``"warning"``, ``"failed"``, or
``"unknown"`` (see :func:`smart_health`); a smartctl call that failed is
logged as a warning and leaves the disk ``"unknown"``
* pool (string) - md array name if member; empty string otherwise
Example::
@@ -255,19 +301,13 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver):
"pool": "md1",
},
]
:raises CommandErrorException: when lsblk printed no device list.
"""
result: list[PhysicalDiskDict] = []
try:
raw = self._send(
"lsblk -J -b -o NAME,SIZE,TYPE,ROTA,VENDOR,MODEL,SERIAL,TRAN 2>/dev/null"
)
lsblk = _json.loads(raw)
except Exception:
return result
devices = self._lsblk()
members = self._md_members()
for dev in lsblk.get("blockdevices", []):
for dev in devices:
if dev.get("type") != "disk":
continue
name = dev.get("name", "")
@@ -307,28 +347,56 @@ class OpenMediaVaultDriver(OmvLocalAccountsMixin, StorageDriver, LinuxDriver):
return result
def _smart(self, name: str, disk_type: str) -> tuple[str, int, int]:
"""A disk's SMART health, temperature (``-1`` if unknown) and rpm (``0``)."""
health, temperature, rpm = "unknown", -1, 0
def _lsblk(self) -> list[dict[str, Any]]:
"""The block devices lsblk lists; a failed or garbled read raises."""
raw = self._send(_LSBLK)
try:
smart = self._sudo(_SMARTCTL.format(name=name), read_timeout=15)
health = smart_health(smart)
temperature = smart_temperature(smart)
if disk_type == "hdd":
rpm_m = re.search(r"Rotation Rate:\s+(\d+)\s+rpm", smart)
if rpm_m:
rpm = int(rpm_m.group(1))
except Exception:
logger.debug("%s: smartctl for %s: ignored", self.hostname, name, exc_info=True)
return health, temperature, rpm
devices = _json.loads(raw)["blockdevices"]
except (ValueError, KeyError, TypeError) as exc:
raise CommandErrorException(
f"lsblk on {self.hostname} printed no device list: {raw[-300:]!r}"
) from exc
if not isinstance(devices, list):
raise CommandErrorException(f"lsblk on {self.hostname}: blockdevices is no list")
return devices
def _smart(self, name: str, disk_type: str) -> tuple[str, int, int]:
"""A disk's SMART health, temperature (``-1`` if unknown) and rpm (``0``).
A smartctl call that failed (timeout, sudo refused, device not opened) is
logged as a warning, so it is told apart from a disk without SMART; the
disk stays in the inventory as ``"unknown"``.
"""
try:
smart, status = self._sudo_status(
_SMARTCTL.format(name=name), read_timeout=_SMARTCTL_TIMEOUT
)
except OSError as exc: # TimeoutError and the socket's own errors
logger.warning("%s: smartctl for %s failed: %s", self.hostname, name, exc)
return "unknown", -1, 0
if status is not None and status & _SMARTCTL_CALL_FAILED:
logger.warning(
"%s: smartctl for %s failed (exit status %s): %s",
self.hostname,
name,
status,
smart[-300:],
)
return "unknown", -1, 0
rpm = 0
if disk_type == "hdd":
rpm_m = re.search(r"Rotation Rate:\s+(\d+)\s+rpm", smart)
if rpm_m:
rpm = int(rpm_m.group(1))
return smart_health(smart), smart_temperature(smart), rpm
def _md_members(self) -> dict[str, str]:
"""The md array of each member device, from one read of ``/proc/mdstat``."""
try:
return md_members(self._send("cat /proc/mdstat 2>/dev/null"))
except Exception:
logger.debug("%s: /proc/mdstat not read: ignored", self.hostname, exc_info=True)
return {}
"""The md array of each member device, from one read of ``/proc/mdstat``.
A host without the md driver has no ``/proc/mdstat`` and no members; a
failed read raises.
"""
return md_members(self._send("cat /proc/mdstat 2>/dev/null"))
# ── md RAID pools ──────────────────────────────────────────────────────────
+2 -2
View File
@@ -1,11 +1,11 @@
{
"coverage_floor": 15.0,
"function_complexity": {
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 12
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 11
},
"function_lines": {
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disk_pools": 81,
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 87,
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_disks": 83,
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_storage_services": 58,
"napalm_openmediavault/openmediavault.py::OpenMediaVaultDriver.get_volumes": 90
},
+158 -8
View File
@@ -1,4 +1,4 @@
"""get_disks: the overall SMART verdict, and the md array each disk belongs to.
"""get_disks: the SMART health, the md array each disk belongs to, and failed reads.
smartctl prints the overall verdict only for ``-H``; ``-i`` (identity) and ``-A``
(attributes) never do. The fake host below answers smartctl with a recorded
@@ -13,10 +13,13 @@ numbers, WWNs and LU ids replaced.
from __future__ import annotations
import json
import logging
import re
from pathlib import Path
from typing import Any
import pytest
from napalm.base.exceptions import CommandErrorException
from napalm_openmediavault.openmediavault import OpenMediaVaultDriver, smart_health
@@ -52,6 +55,39 @@ SAS_PREDICTED = _replaced(
"SMART Health Status: OK\n",
"SMART Health Status: FIRMWARE IMPENDING FAILURE TOO MANY BLOCK REASSIGNS [asc=5d, ascq=64]\n",
)
# No degraded disk was at hand either: these put a count in place of a recorded zero.
_ATA_REALLOCATED = (
" 5 Reallocated_Sector_Ct 0x0000 100 100 000 Old_age Offline - 0\n"
)
ATA_REALLOCATED = _replaced(
ATA_PASSED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8")
)
_ATA_PENDING = (
"197 Current_Pending_Sector 0x0000 100 100 000 Old_age Offline - 0\n"
)
ATA_PENDING = _replaced(ATA_PASSED, _ATA_PENDING, _ATA_PENDING.replace("- 0", "- 2"))
_ATA_OFFLINE_UNC = (
"198 Offline_Uncorrectable 0x0000 100 100 000 Old_age Offline - 0\n"
)
ATA_OFFLINE_UNC = _replaced(
ATA_PASSED, _ATA_OFFLINE_UNC, _ATA_OFFLINE_UNC.replace("- 0", "- 1")
)
ATA_FAILED_REALLOCATED = _replaced(
ATA_FAILED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8")
)
NVME_CRITICAL_WARNING = _replaced(
NVME_PASSED,
"Critical Warning: 0x00\n",
"Critical Warning: 0x04\n",
)
NVME_MEDIA_ERRORS = _replaced(
NVME_PASSED,
"Media and Data Integrity Errors: 0\n",
"Media and Data Integrity Errors: 3\n",
)
SAS_GROWN_DEFECTS = _replaced(
SAS_OK, "Elements in grown defect list: 0\n", "Elements in grown defect list: 12\n"
)
#: What ``smartctl -iA`` prints: no verdict, but the attribute table's WHEN_FAILED header.
ATA_WITHOUT_VERDICT = _replaced(ATA_PASSED, _ATA_VERDICT, "")
@@ -66,16 +102,29 @@ def smartctl(flags: str, recording: str) -> str:
class Nas:
"""The NAS as ``_send`` and ``_sudo`` see it: lsblk, /proc/mdstat and smartctl."""
"""The NAS as ``_send`` and ``_sudo_status`` see it: lsblk, /proc/mdstat, smartctl.
def __init__(self, disks: dict[str, tuple[str, str]], mdstat: str = "") -> None:
A disk's smartctl answer is its recorded output (exit status 0), an
``(output, exit status)`` pair, or an exception the transport raises. *lsblk* and
*mdstat* given as an exception are raised the same way.
"""
def __init__(
self,
disks: dict[str, tuple[str, Any]],
mdstat: str | Exception = "",
lsblk: str | None = None,
) -> None:
#: name -> (tran, recorded ``smartctl -iHA`` output)
self.disks = disks
self.mdstat = mdstat
self.lsblk = lsblk
self.commands: list[str] = []
def send(self, command: str, read_timeout: float = 100) -> str:
self.commands.append(command)
if command.startswith("lsblk ") and self.lsblk is not None:
return self.lsblk
if command.startswith("lsblk "):
return json.dumps(
{
@@ -84,7 +133,7 @@ class Nas:
"name": name,
"size": 6001175126016,
"type": "disk",
"rota": tran != "nvme" and "Solid State Device" not in recording,
"rota": tran != "nvme" and "Solid State Device" not in str(recording),
"vendor": "",
"model": "m",
"serial": "s",
@@ -95,21 +144,26 @@ class Nas:
}
)
if command.startswith("cat /proc/mdstat"):
if isinstance(self.mdstat, Exception):
raise self.mdstat
return self.mdstat
raise AssertionError(command)
def sudo(self, command: str, read_timeout: float = 100) -> str:
def sudo_status(self, command: str, read_timeout: float = 100) -> tuple[str, int | None]:
self.commands.append(command)
m = re.match(r"smartctl -(\w+) /dev/(\w+)", command)
assert m, command
_tran, recording = self.disks[m.group(2)]
return smartctl(m.group(1), recording)
_tran, answer = self.disks[m.group(2)]
if isinstance(answer, Exception):
raise answer
output, status = answer if isinstance(answer, tuple) else (answer, 0)
return smartctl(m.group(1), output), status
def _driver(nas: Nas) -> OpenMediaVaultDriver:
driver = OpenMediaVaultDriver("nas", "admin", "p")
driver._send = nas.send # type: ignore[method-assign]
driver._sudo = nas.sudo # type: ignore[method-assign]
driver._sudo_status = nas.sudo_status # type: ignore[method-assign]
return driver
@@ -173,6 +227,15 @@ def test_a_disk_without_smart_has_no_temperature():
("", "unknown"),
# The attribute table's WHEN_FAILED column is no verdict.
(ATA_WITHOUT_VERDICT, "unknown"),
# A passing disk that is losing sectors is degraded (#16).
(ATA_REALLOCATED, "warning"),
(ATA_PENDING, "warning"),
(ATA_OFFLINE_UNC, "warning"),
(NVME_CRITICAL_WARNING, "warning"),
(NVME_MEDIA_ERRORS, "warning"),
(SAS_GROWN_DEFECTS, "warning"),
# The verdict outranks the counters.
(ATA_FAILED_REALLOCATED, "failed"),
],
ids=[
"ata-passed",
@@ -183,12 +246,99 @@ def test_a_disk_without_smart_has_no_temperature():
"no-smart",
"empty",
"when-failed-column",
"ata-reallocated",
"ata-pending",
"ata-offline-uncorrectable",
"nvme-critical-warning",
"nvme-media-errors",
"sas-grown-defects",
"ata-failed-and-reallocated",
],
)
def test_smart_health_reads_the_verdict_line_only(output, health):
assert smart_health(output) == health
def test_a_degraded_disk_is_reported_as_warning():
nas = Nas({"sda": ("sata", ATA_REALLOCATED)})
assert _by_slot(nas)["/dev/sda"]["health"] == "warning"
# ── failed reads (#16) ──────────────────────────────────────────────────────────
@pytest.mark.parametrize(
"lsblk",
[
"",
"sudo: a terminal is required to read the password",
"lsblk: unknown column: TRAN",
'["not", "an", "object"]',
],
ids=["nothing", "sudo", "lsblk-error", "not-an-object"],
)
def test_a_failed_lsblk_raises(lsblk):
"""``[]`` is a NAS without disks; a failed read must not look like one."""
nas = Nas({}, lsblk=lsblk)
with pytest.raises(CommandErrorException, match="lsblk"):
_driver(nas).get_disks()
def test_a_nas_without_disks_has_none():
nas = Nas({}, lsblk='{"blockdevices": []}')
assert _driver(nas).get_disks() == []
def test_a_smartctl_timeout_is_logged_as_a_warning(caplog):
nas = Nas(
{
"sda": ("sata", TimeoutError("'smartctl -iHA /dev/sda' did not finish within 15s")),
"sdb": ("sata", ATA_PASSED),
}
)
with caplog.at_level(logging.WARNING):
disks = _by_slot(nas)
assert (disks["/dev/sda"]["health"], disks["/dev/sda"]["temperature"]) == ("unknown", -1)
assert disks["/dev/sdb"]["health"] == "healthy"
warned = [r for r in caplog.records if r.levelno == logging.WARNING]
assert any("sda" in r.getMessage() and "smartctl" in r.getMessage() for r in warned)
def test_a_refused_sudo_for_smartctl_is_logged_as_a_warning(caplog):
# sudo's own refusal exits 1, which smartctl uses for "command line did not parse".
nas = Nas({"sda": ("sata", ("sudo: a password is required", 1))})
with caplog.at_level(logging.WARNING):
disk = _by_slot(nas)["/dev/sda"]
assert disk["health"] == "unknown"
warned = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
assert any("sda" in m and "sudo: a password is required" in m for m in warned)
def test_a_disk_status_in_smartctls_exit_code_is_no_failed_call(caplog):
# Bits 3-7 of smartctl's exit status describe the disk, not the call.
nas = Nas({"sda": ("sata", (ATA_FAILED, 8))})
with caplog.at_level(logging.WARNING):
disk = _by_slot(nas)["/dev/sda"]
assert disk["health"] == "failed"
assert not [r for r in caplog.records if r.levelno >= logging.WARNING]
def test_a_failed_mdstat_read_raises():
nas = Nas({"sda": ("sata", ATA_PASSED)}, mdstat=TimeoutError("no data within 100s"))
with pytest.raises(TimeoutError):
_driver(nas).get_disks()
# ── md membership ───────────────────────────────────────────────────────────────
# The recorded hosts have no md array (MDSTAT_NO_ARRAYS), so the arrays below are