driver / standard (pull_request) Successful in 21s
driver / lint (pull_request) Successful in 18s
driver / types (pull_request) Successful in 1m17s
driver / test (3.12) (pull_request) Successful in 1m12s
driver / test (3.13) (pull_request) Successful in 1m7s
driver / lowest (pull_request) Successful in 42s
driver / build (pull_request) Successful in 51s
driver / security (pull_request) Successful in 1m35s
CI / driver (pull_request) Successful in 3m26s
driver / publish (pull_request) Skipped
get_disks swallowed a failed or garbled lsblk and returned [], the answer for a NAS without disks, so every disk vanished on a transient error. It now raises CommandErrorException; a failed /proc/mdstat read raises too instead of dropping every pool. A smartctl call that timed out, was refused by sudo or could not open the device was logged at debug level and read like a disk without SMART. It is now logged as a warning with the reason (exit status bits 0 and 1; the disk-state bits 2-7 are not a failed call). A disk whose verdict passed but that counts reallocated, pending or offline-uncorrectable sectors (ATA), grown defects (SAS), or sets a critical warning or counts media errors (NVMe) is reported as "warning", as the contract allows. get_disks shrinks below its baseline entries, which are tightened. Closes #16
393 lines
13 KiB
Python
393 lines
13 KiB
Python
"""get_disks: the SMART health, the md array each disk belongs to, and failed reads.
|
|
|
|
smartctl prints the overall verdict only for ``-H``; ``-i`` (identity) and ``-A``
|
|
(attributes) never do. The fake host below answers smartctl with a recorded
|
|
``smartctl -iHA`` output and drops the verdict line when the command does not ask
|
|
for ``-H``.
|
|
|
|
The recordings in ``fixtures/`` are smartmontools 7.5 on Proxmox VE (an ATA SATA DOM,
|
|
a SAS SSD, an NVMe SSD) and 7.4 on an OMV VM (a QEMU disk without SMART), with serial
|
|
numbers, WWNs and LU ids replaced.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from napalm.base.exceptions import CommandErrorException
|
|
|
|
from napalm_openmediavault.openmediavault import OpenMediaVaultDriver, smart_health
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def _recorded(name: str) -> str:
|
|
return (FIXTURES / name).read_text(encoding="utf-8")
|
|
|
|
|
|
def _replaced(recording: str, line: str, by: str) -> str:
|
|
"""The recording with one of its lines replaced; the line must be there."""
|
|
assert recording.count(line) == 1, line
|
|
return recording.replace(line, by)
|
|
|
|
|
|
ATA_PASSED = _recorded("smartctl-7.5-ata-supermicro-superdom-SOB20R-passed.txt")
|
|
NVME_PASSED = _recorded("smartctl-7.5-nvme-crucial-CT500P5PSSD8-P7CR402-passed.txt")
|
|
SAS_OK = _recorded("smartctl-7.5-sas-hgst-HSCAC2DA4SUN400G-A29A-ok.txt")
|
|
QEMU_NO_SMART = _recorded("smartctl-7.4-scsi-qemu-harddisk-unavailable.txt")
|
|
|
|
_ATA_VERDICT = "SMART overall-health self-assessment test result: PASSED\n"
|
|
# No failing disk was at hand: the failing verdicts are smartmontools' wording put in
|
|
# place of a recorded verdict line.
|
|
ATA_FAILED = _replaced(
|
|
ATA_PASSED,
|
|
_ATA_VERDICT,
|
|
"SMART overall-health self-assessment test result: FAILED!\n"
|
|
"Drive failure expected in less than 24 hours. SAVE ALL DATA.\n",
|
|
)
|
|
SAS_PREDICTED = _replaced(
|
|
SAS_OK,
|
|
"SMART Health Status: OK\n",
|
|
"SMART Health Status: FIRMWARE IMPENDING FAILURE TOO MANY BLOCK REASSIGNS [asc=5d, ascq=64]\n",
|
|
)
|
|
# No degraded disk was at hand either: these put a count in place of a recorded zero.
|
|
_ATA_REALLOCATED = (
|
|
" 5 Reallocated_Sector_Ct 0x0000 100 100 000 Old_age Offline - 0\n"
|
|
)
|
|
ATA_REALLOCATED = _replaced(
|
|
ATA_PASSED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8")
|
|
)
|
|
_ATA_PENDING = (
|
|
"197 Current_Pending_Sector 0x0000 100 100 000 Old_age Offline - 0\n"
|
|
)
|
|
ATA_PENDING = _replaced(ATA_PASSED, _ATA_PENDING, _ATA_PENDING.replace("- 0", "- 2"))
|
|
_ATA_OFFLINE_UNC = (
|
|
"198 Offline_Uncorrectable 0x0000 100 100 000 Old_age Offline - 0\n"
|
|
)
|
|
ATA_OFFLINE_UNC = _replaced(
|
|
ATA_PASSED, _ATA_OFFLINE_UNC, _ATA_OFFLINE_UNC.replace("- 0", "- 1")
|
|
)
|
|
ATA_FAILED_REALLOCATED = _replaced(
|
|
ATA_FAILED, _ATA_REALLOCATED, _ATA_REALLOCATED.replace("- 0", "- 8")
|
|
)
|
|
NVME_CRITICAL_WARNING = _replaced(
|
|
NVME_PASSED,
|
|
"Critical Warning: 0x00\n",
|
|
"Critical Warning: 0x04\n",
|
|
)
|
|
NVME_MEDIA_ERRORS = _replaced(
|
|
NVME_PASSED,
|
|
"Media and Data Integrity Errors: 0\n",
|
|
"Media and Data Integrity Errors: 3\n",
|
|
)
|
|
SAS_GROWN_DEFECTS = _replaced(
|
|
SAS_OK, "Elements in grown defect list: 0\n", "Elements in grown defect list: 12\n"
|
|
)
|
|
#: What ``smartctl -iA`` prints: no verdict, but the attribute table's WHEN_FAILED header.
|
|
ATA_WITHOUT_VERDICT = _replaced(ATA_PASSED, _ATA_VERDICT, "")
|
|
|
|
_VERDICT_LINE_RE = re.compile(
|
|
r"^SMART (?:overall-health self-assessment test result|Health Status):.*\n", re.M
|
|
)
|
|
|
|
|
|
def smartctl(flags: str, recording: str) -> str:
|
|
"""What ``smartctl -<flags>`` prints: the recording, its verdict only for ``-H``."""
|
|
return recording if "H" in flags else _VERDICT_LINE_RE.sub("", recording)
|
|
|
|
|
|
class Nas:
|
|
"""The NAS as ``_send`` and ``_sudo_status`` see it: lsblk, /proc/mdstat, smartctl.
|
|
|
|
A disk's smartctl answer is its recorded output (exit status 0), an
|
|
``(output, exit status)`` pair, or an exception the transport raises. *lsblk* and
|
|
*mdstat* given as an exception are raised the same way.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
disks: dict[str, tuple[str, Any]],
|
|
mdstat: str | Exception = "",
|
|
lsblk: str | None = None,
|
|
) -> None:
|
|
#: name -> (tran, recorded ``smartctl -iHA`` output)
|
|
self.disks = disks
|
|
self.mdstat = mdstat
|
|
self.lsblk = lsblk
|
|
self.commands: list[str] = []
|
|
|
|
def send(self, command: str, read_timeout: float = 100) -> str:
|
|
self.commands.append(command)
|
|
if command.startswith("lsblk ") and self.lsblk is not None:
|
|
return self.lsblk
|
|
if command.startswith("lsblk "):
|
|
return json.dumps(
|
|
{
|
|
"blockdevices": [
|
|
{
|
|
"name": name,
|
|
"size": 6001175126016,
|
|
"type": "disk",
|
|
"rota": tran != "nvme" and "Solid State Device" not in str(recording),
|
|
"vendor": "",
|
|
"model": "m",
|
|
"serial": "s",
|
|
"tran": tran,
|
|
}
|
|
for name, (tran, recording) in self.disks.items()
|
|
]
|
|
}
|
|
)
|
|
if command.startswith("cat /proc/mdstat"):
|
|
if isinstance(self.mdstat, Exception):
|
|
raise self.mdstat
|
|
return self.mdstat
|
|
raise AssertionError(command)
|
|
|
|
def sudo_status(self, command: str, read_timeout: float = 100) -> tuple[str, int | None]:
|
|
self.commands.append(command)
|
|
m = re.match(r"smartctl -(\w+) /dev/(\w+)", command)
|
|
assert m, command
|
|
_tran, answer = self.disks[m.group(2)]
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
output, status = answer if isinstance(answer, tuple) else (answer, 0)
|
|
return smartctl(m.group(1), output), status
|
|
|
|
|
|
def _driver(nas: Nas) -> OpenMediaVaultDriver:
|
|
driver = OpenMediaVaultDriver("nas", "admin", "p")
|
|
driver._send = nas.send # type: ignore[method-assign]
|
|
driver._sudo_status = nas.sudo_status # type: ignore[method-assign]
|
|
return driver
|
|
|
|
|
|
def _by_slot(nas: Nas) -> dict[str, dict]:
|
|
return {d["slot"]: dict(d) for d in _driver(nas).get_disks()}
|
|
|
|
|
|
# ── the overall verdict ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_a_passing_ata_disk_is_healthy():
|
|
nas = Nas({"sda": ("sata", ATA_PASSED)})
|
|
|
|
disk = _by_slot(nas)["/dev/sda"]
|
|
|
|
assert disk["health"] == "healthy"
|
|
# A solid-state disk has no rotation rate.
|
|
assert (disk["temperature"], disk["rpm"]) == (34, 0)
|
|
|
|
|
|
def test_a_failing_ata_disk_is_failed():
|
|
nas = Nas({"sda": ("sata", ATA_FAILED)})
|
|
|
|
assert _by_slot(nas)["/dev/sda"]["health"] == "failed"
|
|
|
|
|
|
def test_a_passing_nvme_disk_is_healthy():
|
|
nas = Nas({"nvme0n1": ("nvme", NVME_PASSED)})
|
|
|
|
disk = _by_slot(nas)["/dev/nvme0n1"]
|
|
|
|
assert (disk["health"], disk["temperature"]) == ("healthy", 40)
|
|
|
|
|
|
def test_a_sas_disk_reports_its_drive_temperature():
|
|
nas = Nas({"sdb": ("", SAS_OK)})
|
|
|
|
disk = _by_slot(nas)["/dev/sdb"]
|
|
|
|
assert (disk["health"], disk["temperature"]) == ("healthy", 28)
|
|
|
|
|
|
def test_a_disk_without_smart_has_no_temperature():
|
|
# QEMU prints "Current Drive Temperature: 0 C" for a disk that has no sensor.
|
|
nas = Nas({"sda": ("", QEMU_NO_SMART)})
|
|
|
|
disk = _by_slot(nas)["/dev/sda"]
|
|
|
|
assert (disk["health"], disk["temperature"]) == ("unknown", -1)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("output", "health"),
|
|
[
|
|
(ATA_PASSED, "healthy"),
|
|
(ATA_FAILED, "failed"),
|
|
(NVME_PASSED, "healthy"),
|
|
(SAS_OK, "healthy"),
|
|
(SAS_PREDICTED, "failed"),
|
|
(QEMU_NO_SMART, "unknown"),
|
|
("", "unknown"),
|
|
# The attribute table's WHEN_FAILED column is no verdict.
|
|
(ATA_WITHOUT_VERDICT, "unknown"),
|
|
# A passing disk that is losing sectors is degraded (#16).
|
|
(ATA_REALLOCATED, "warning"),
|
|
(ATA_PENDING, "warning"),
|
|
(ATA_OFFLINE_UNC, "warning"),
|
|
(NVME_CRITICAL_WARNING, "warning"),
|
|
(NVME_MEDIA_ERRORS, "warning"),
|
|
(SAS_GROWN_DEFECTS, "warning"),
|
|
# The verdict outranks the counters.
|
|
(ATA_FAILED_REALLOCATED, "failed"),
|
|
],
|
|
ids=[
|
|
"ata-passed",
|
|
"ata-failed",
|
|
"nvme-passed",
|
|
"sas-ok",
|
|
"sas-predicted",
|
|
"no-smart",
|
|
"empty",
|
|
"when-failed-column",
|
|
"ata-reallocated",
|
|
"ata-pending",
|
|
"ata-offline-uncorrectable",
|
|
"nvme-critical-warning",
|
|
"nvme-media-errors",
|
|
"sas-grown-defects",
|
|
"ata-failed-and-reallocated",
|
|
],
|
|
)
|
|
def test_smart_health_reads_the_verdict_line_only(output, health):
|
|
assert smart_health(output) == health
|
|
|
|
|
|
def test_a_degraded_disk_is_reported_as_warning():
|
|
nas = Nas({"sda": ("sata", ATA_REALLOCATED)})
|
|
|
|
assert _by_slot(nas)["/dev/sda"]["health"] == "warning"
|
|
|
|
|
|
# ── failed reads (#16) ──────────────────────────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"lsblk",
|
|
[
|
|
"",
|
|
"sudo: a terminal is required to read the password",
|
|
"lsblk: unknown column: TRAN",
|
|
'["not", "an", "object"]',
|
|
],
|
|
ids=["nothing", "sudo", "lsblk-error", "not-an-object"],
|
|
)
|
|
def test_a_failed_lsblk_raises(lsblk):
|
|
"""``[]`` is a NAS without disks; a failed read must not look like one."""
|
|
nas = Nas({}, lsblk=lsblk)
|
|
|
|
with pytest.raises(CommandErrorException, match="lsblk"):
|
|
_driver(nas).get_disks()
|
|
|
|
|
|
def test_a_nas_without_disks_has_none():
|
|
nas = Nas({}, lsblk='{"blockdevices": []}')
|
|
|
|
assert _driver(nas).get_disks() == []
|
|
|
|
|
|
def test_a_smartctl_timeout_is_logged_as_a_warning(caplog):
|
|
nas = Nas(
|
|
{
|
|
"sda": ("sata", TimeoutError("'smartctl -iHA /dev/sda' did not finish within 15s")),
|
|
"sdb": ("sata", ATA_PASSED),
|
|
}
|
|
)
|
|
|
|
with caplog.at_level(logging.WARNING):
|
|
disks = _by_slot(nas)
|
|
|
|
assert (disks["/dev/sda"]["health"], disks["/dev/sda"]["temperature"]) == ("unknown", -1)
|
|
assert disks["/dev/sdb"]["health"] == "healthy"
|
|
warned = [r for r in caplog.records if r.levelno == logging.WARNING]
|
|
assert any("sda" in r.getMessage() and "smartctl" in r.getMessage() for r in warned)
|
|
|
|
|
|
def test_a_refused_sudo_for_smartctl_is_logged_as_a_warning(caplog):
|
|
# sudo's own refusal exits 1, which smartctl uses for "command line did not parse".
|
|
nas = Nas({"sda": ("sata", ("sudo: a password is required", 1))})
|
|
|
|
with caplog.at_level(logging.WARNING):
|
|
disk = _by_slot(nas)["/dev/sda"]
|
|
|
|
assert disk["health"] == "unknown"
|
|
warned = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
|
assert any("sda" in m and "sudo: a password is required" in m for m in warned)
|
|
|
|
|
|
def test_a_disk_status_in_smartctls_exit_code_is_no_failed_call(caplog):
|
|
# Bits 3-7 of smartctl's exit status describe the disk, not the call.
|
|
nas = Nas({"sda": ("sata", (ATA_FAILED, 8))})
|
|
|
|
with caplog.at_level(logging.WARNING):
|
|
disk = _by_slot(nas)["/dev/sda"]
|
|
|
|
assert disk["health"] == "failed"
|
|
assert not [r for r in caplog.records if r.levelno >= logging.WARNING]
|
|
|
|
|
|
def test_a_failed_mdstat_read_raises():
|
|
nas = Nas({"sda": ("sata", ATA_PASSED)}, mdstat=TimeoutError("no data within 100s"))
|
|
|
|
with pytest.raises(TimeoutError):
|
|
_driver(nas).get_disks()
|
|
|
|
|
|
# ── md membership ───────────────────────────────────────────────────────────────
|
|
|
|
# The recorded hosts have no md array (MDSTAT_NO_ARRAYS), so the arrays below are
|
|
# written out by hand to cover look-alike names and a failed member.
|
|
MDSTAT = """\
|
|
Personalities : [raid1] [linear] [multipath] [raid0] [raid6] [raid5] [raid4] [raid10]
|
|
md0 : active raid1 sdab1[1] sdaa1[0]
|
|
976630464 blocks super 1.2 [2/2] [UU]
|
|
|
|
md1 : active raid1 sdb1[1] sdc1[0](F)
|
|
5860389888 blocks super 1.2 [2/1] [U_]
|
|
|
|
md2 : active raid1 nvme0n1p1[0] nvme1n1[1]
|
|
976630464 blocks super 1.2 [2/2] [UU]
|
|
|
|
md3 : active raid1 nvme2n11[0] nvme2n12[1]
|
|
976630464 blocks super 1.2 [2/2] [UU]
|
|
|
|
unused devices: <none>
|
|
"""
|
|
MDSTAT_NO_ARRAYS = _recorded("mdstat-omv-no-arrays.txt")
|
|
|
|
|
|
def test_a_disk_is_in_the_array_its_partition_or_itself_is_in():
|
|
names = ["sda", "sdaa", "sdc", "nvme0n1", "nvme1n1", "nvme2n1"]
|
|
nas = Nas({n: ("sata", ATA_PASSED) for n in names}, MDSTAT)
|
|
|
|
pools = {slot: d["pool"] for slot, d in _by_slot(nas).items()}
|
|
|
|
assert pools == {
|
|
"/dev/sda": "", # sdaa1 and sdab1 are not sda's
|
|
"/dev/sdaa": "md0",
|
|
"/dev/sdc": "md1", # a failed member is still a member
|
|
"/dev/nvme0n1": "md2",
|
|
"/dev/nvme1n1": "md2",
|
|
"/dev/nvme2n1": "", # nvme2n11 is another namespace, not a partition
|
|
}
|
|
|
|
|
|
def test_no_disk_is_in_an_array_when_there_is_none():
|
|
nas = Nas({n: ("", QEMU_NO_SMART) for n in ("sda", "sdb")}, MDSTAT_NO_ARRAYS)
|
|
|
|
assert {d["pool"] for d in _by_slot(nas).values()} == {""}
|
|
|
|
|
|
def test_mdstat_is_read_once_for_all_disks():
|
|
nas = Nas({n: ("sata", ATA_PASSED) for n in ("sda", "sdb")}, MDSTAT)
|
|
|
|
_driver(nas).get_disks()
|
|
|
|
assert sum(c.startswith("cat /proc/mdstat") for c in nas.commands) == 1
|