diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a39102..9d48c97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ recorded here; their history is the git log. on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). - The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python 3.12 venv no longer has. +- The SNMP health profile shapes (`SnmpHealthProfile`, `SnmpValueSpec`, + `SnmpMemorySpec`, `SnmpInterfacesSpec`) are defined in + `napalm_device_types.snmp_health_profile`. The package and `models` still export + them, so an existing import keeps working. ### Added - `HypervisorDriver.get_vm_migration_preflight(name)`, `migrate_vm(name, target_node, @@ -36,6 +40,14 @@ recorded here; their history is the git log. (`manage_service(LLDP_AGENT_SERVICE, …)`); both names default to `lldpd`. No interface is a ValueError before anything changes. It is the action for what LLDP getters used to do on their own (NAPALM/napalm-openwrt#8). +- `SnmpValueSpec.unknown`: a driver whose device's agent answers a value wrongly says + so in its `SNMP_HEALTH_PROFILE` (`"cpu": {"unknown": True}`), and the application + reports no value for it -- not the standard reading, not 0. OpenBSD's net-snmp, + which reports the CPU as never idle, is the case (NAPALM/napalm-bsd#7). +- `VMProvisionIncomplete` (`napalm_device_types.provisioning`): what + `create_vm_from_cloud_init` raises when it fails after the VM was created. A + `RuntimeError` with the failure as its cause and the half-built VM's `vmid` and + `node`, so the caller can remove it with `destroy_vm` (NAPALM/napalm-proxmox#28). - `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, with its nodes (offline ones included, `local` on the node the connection is), diff --git a/README.md b/README.md index 54a6d9a..9c59ae8 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,9 @@ class MySwitchDriver(SwitchDriver): A value is `oids` (the first that answers wins) or a `table` folded by `first`, `average` or `max`, with an optional `pattern` for an agent that answers with text. What the profile does not name, the application reads from the standard UCD-MIB and IF-MIB. +A value the agent gets wrong is `{"unknown": True}`: the application then reports no value +for it, rather than the standard reading or a default (OpenBSD's net-snmp reports its CPU +as never idle). `KernelFactsMixin` (`get_kernel_facts`) is the exception that is mixed in by a driver rather than by a role base: what a Linux kernel has built and loaded is read the same way @@ -548,7 +551,10 @@ any other. A QEMU-based driver takes `provisioning.QEMU_GUEST_AGENTS` (Linux, FreeBSD, OpenBSD). `provisioning.network_config()` writes the network-config v2 every cloud-init flavour reads (FreeBSD's nuageinit reads no other), and `provisioning.split_compression()` says how a packed image is -unpacked; both are the same for every hypervisor. +unpacked; both are the same for every hypervisor. A creation that fails once +the VM exists raises `VMProvisionIncomplete`, a `RuntimeError` that names the +half-built VM (`vmid`, `node`) for the caller to `destroy_vm`, with the failure +as its cause. ### OS / Linux diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index f274558..68f28d1 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -164,10 +164,6 @@ from napalm_device_types.models import ( PowerAction, PowerStateDict, SnmpAgentSettingsDict, - SnmpHealthProfile, - SnmpInterfacesSpec, - SnmpMemorySpec, - SnmpValueSpec, UpdateProgressDict, ) from napalm_device_types.nat_vpn import NatVpnMixin @@ -182,6 +178,7 @@ from napalm_device_types.package_updates import ( from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.phone import PhoneDriver from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping +from napalm_device_types.provisioning import VMProvisionIncomplete from napalm_device_types.residential_gateway import ResidentialGatewayDriver from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of from napalm_device_types.service_clusters import ( @@ -196,6 +193,12 @@ from napalm_device_types.service_packages import ( parse_service_packages, ) from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.snmp_health_profile import ( + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, +) from napalm_device_types.ssh_host_keys import ( KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, ) @@ -363,6 +366,7 @@ __all__ = [ "VIRTUAL_IPS_COMMAND", "VirtualIpsMixin", "VmStopRequestMixin", + "VMProvisionIncomplete", "SERVICE_CLUSTERS_COMMAND", "ServiceClustersMixin", "parse_service_clusters", diff --git a/napalm_device_types/health_metrics.py b/napalm_device_types/health_metrics.py index 5066475..de0bd45 100644 --- a/napalm_device_types/health_metrics.py +++ b/napalm_device_types/health_metrics.py @@ -2,8 +2,8 @@ **Deprecated** (docs/STRUCTURE.md): drivers do not collect over SNMP. The application polls; a driver whose device publishes values under its own OIDs -declares them as ``SNMP_HEALTH_PROFILE`` (:class:`~napalm_device_types.models.SnmpHealthProfile`). -This mixin goes in 5.0. +declares them as ``SNMP_HEALTH_PROFILE`` +(:class:`~napalm_device_types.snmp_health_profile.SnmpHealthProfile`). This mixin goes in 5.0. The collection itself is generic: CPU, memory, uptime, load and per-interface counters come from the same standard OIDs on any UCD-MIB/IF-MIB capable device. diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 7b2a8d0..8784884 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -473,13 +473,11 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ Create a new virtual machine from a cloud image via Cloud-Init. - Downloads the cloud image (qcow2/raw) directly on the hypervisor if not - already cached there, creates a new VM shell, imports the image as its - root disk, configures virtual network interfaces, and injects Cloud-Init - configuration via a storage snippet or similar mechanism. The resulting - VM is left in a running state. - - Implementations cache downloaded images on the hypervisor by URL/filename. + Downloads the cloud image (qcow2/raw) directly on the hypervisor unless it is + cached there already (implementations cache by URL/filename), creates a new VM + shell, imports the image as its root disk, configures virtual network interfaces, + and injects Cloud-Init configuration via a storage snippet or similar mechanism. + The resulting VM is left in a running state. Args: name (string) - new VM display name @@ -533,6 +531,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri Raises: RuntimeError - if provisioning fails (download failure, checksum mismatch, storage unavailable, invalid config, timeout, etc.) + VMProvisionIncomplete - a RuntimeError for a failure after the VM was created, + with that VM's ``vmid`` and ``node`` for ``destroy_vm`` and the failure as cause. """ ... diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 112a57c..b791092 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -9,6 +9,12 @@ from typing import Literal, NotRequired, get_args from typing_extensions import TypedDict +# The SNMP health profile shapes live in snmp_health_profile.py; re-exported here, where they were. +from napalm_device_types.snmp_health_profile import SnmpHealthProfile as SnmpHealthProfile +from napalm_device_types.snmp_health_profile import SnmpInterfacesSpec as SnmpInterfacesSpec +from napalm_device_types.snmp_health_profile import SnmpMemorySpec as SnmpMemorySpec +from napalm_device_types.snmp_health_profile import SnmpValueSpec as SnmpValueSpec + # --------------------------------------------------------------------------- # Common (shared across device types) # --------------------------------------------------------------------------- @@ -1230,53 +1236,6 @@ class SnmpAgentSettingsDict(TypedDict): listen: NotRequired[str] # the address the agent binds to; absent: every address -class SnmpValueSpec(TypedDict, total=False): - """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. - - Either ``oids``, scalars asked in order (the first non-empty answer wins), or - ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a - regular expression whose first group is the number, for an agent that answers - with text. - """ - - oids: list[str] - table: str - fold: Literal["first", "average", "max"] - pattern: str - - -class SnmpMemorySpec(TypedDict, total=False): - """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" - - total: SnmpValueSpec - used: SnmpValueSpec - free: SnmpValueSpec - percent: SnmpValueSpec - unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default - - -class SnmpInterfacesSpec(TypedDict, total=False): - """How the IF-MIB counters of a device are to be read.""" - - skip: str # a pattern of interface names whose counters are noise, not signal - tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) - - -class SnmpHealthProfile(TypedDict, total=False): - """Where a device's SNMP agent publishes its health values. - - A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer - the standard UCD-MIB and IF-MIB; the application polls and computes. What a - profile does not name is read the standard way. Uptime is in TimeTicks, CPU in - percent busy. - """ - - uptime: SnmpValueSpec - cpu: SnmpValueSpec - memory: SnmpMemorySpec - interfaces: SnmpInterfacesSpec - - class NICConfigDict(TypedDict): """Network interface configuration for VM provisioning.""" diff --git a/napalm_device_types/provisioning.py b/napalm_device_types/provisioning.py index 8879934..b479614 100644 --- a/napalm_device_types/provisioning.py +++ b/napalm_device_types/provisioning.py @@ -3,8 +3,9 @@ What provisioning a VM from a cloud image needs to know about the guest. The same for every hypervisor: which agent a guest runs so a QEMU-based hypervisor can read its IP, the network-config cloud-init and FreeBSD's -nuageinit both read, and how a packed cloud image is unpacked. How a -hypervisor attaches, boots and talks to the VM stays in its driver. +nuageinit both read, how a packed cloud image is unpacked, and how a failure +names the VM it left behind. How a hypervisor attaches, boots and talks to +the VM stays in its driver. """ from __future__ import annotations @@ -37,6 +38,30 @@ _DECOMPRESSORS = { } +class VMProvisionIncomplete(RuntimeError): + """``create_vm_from_cloud_init`` failed after the VM was created. + + The VM is still on the hypervisor, half built: ``vmid`` and ``node`` name it + as ``VMProvisionResultDict`` would have, so the caller can remove it with + ``destroy_vm``. The failure itself is the ``__cause__`` (``raise ... from``), + and the message is its own. A failure before the VM exists raises as before. + """ + + def __init__(self, message: str, *, vmid: str, node: str) -> None: + super().__init__(message) + self.vmid = vmid + self.node = node + + def __reduce__(self) -> tuple[Any, ...]: + # The keyword-only arguments are not in ``args``, which is all that + # pickle passes back by default. + return (_vm_provision_incomplete, (str(self), self.vmid, self.node)) + + +def _vm_provision_incomplete(message: str, vmid: str, node: str) -> VMProvisionIncomplete: + return VMProvisionIncomplete(message, vmid=vmid, node=node) + + def network_config(nics: list[tuple[str, bool]]) -> dict[str, Any] | None: """Network-config v2: DHCP on every ``(mac, dhcp)`` NIC that asks for it. diff --git a/napalm_device_types/snmp_health_profile.py b/napalm_device_types/snmp_health_profile.py new file mode 100644 index 0000000..7a01d5f --- /dev/null +++ b/napalm_device_types/snmp_health_profile.py @@ -0,0 +1,60 @@ +"""The shape a driver states its device's SNMP health OIDs in (``SNMP_HEALTH_PROFILE``). + +Apart from ``models.py``, which is over the file size limit; ``models`` still exports them. +""" + +from typing import Literal + +from typing_extensions import TypedDict + + +class SnmpValueSpec(TypedDict, total=False): + """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. + + Either ``oids``, scalars asked in order (the first non-empty answer wins), or + ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a + regular expression whose first group is the number, for an agent that answers + with text. + + ``unknown`` says the agent answers this value wrongly, or not at all, where it + should: the application reports no value -- neither the standard reading nor a + default. It stands alone, without ``oids`` or ``table``. + """ + + oids: list[str] + table: str + fold: Literal["first", "average", "max"] + pattern: str + unknown: bool + + +class SnmpMemorySpec(TypedDict, total=False): + """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" + + total: SnmpValueSpec + used: SnmpValueSpec + free: SnmpValueSpec + percent: SnmpValueSpec + unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default + + +class SnmpInterfacesSpec(TypedDict, total=False): + """How the IF-MIB counters of a device are to be read.""" + + skip: str # a pattern of interface names whose counters are noise, not signal + tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) + + +class SnmpHealthProfile(TypedDict, total=False): + """Where a device's SNMP agent publishes its health values. + + A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer + the standard UCD-MIB and IF-MIB; the application polls and computes. What a + profile does not name is read the standard way, and what it marks ``unknown`` is + not read at all. Uptime is in TimeTicks, CPU in percent busy. + """ + + uptime: SnmpValueSpec + cpu: SnmpValueSpec + memory: SnmpMemorySpec + interfaces: SnmpInterfacesSpec diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index 82c402c..3b89766 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -32,7 +32,7 @@ }, "mypy_relaxed": [], "python_file_lines": { - "napalm_device_types/models.py": 1504 + "napalm_device_types/models.py": 1463 }, "untyped_public": [] } diff --git a/tests/test_snmp_health_profile.py b/tests/test_snmp_health_profile.py index c5b306b..b4f2a10 100644 --- a/tests/test_snmp_health_profile.py +++ b/tests/test_snmp_health_profile.py @@ -22,7 +22,7 @@ from napalm_device_types import ( def test_a_profile_says_where_each_value_lives(): assert set(get_type_hints(SnmpHealthProfile)) == {"uptime", "cpu", "memory", "interfaces"} - assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern"} + assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern", "unknown"} assert set(get_type_hints(SnmpMemorySpec)) == {"total", "used", "free", "percent", "unit"} assert set(get_type_hints(SnmpInterfacesSpec)) == {"skip", "tx_errors_are_drops"} @@ -42,6 +42,24 @@ def test_a_driver_declares_its_profile_as_a_class_attribute(): assert Driver.SNMP_HEALTH_PROFILE["cpu"]["oids"][0] == "1.3.6.1.4.1.9.1" +def test_a_value_the_agent_gets_wrong_is_declared_unknown(): + """Neither the standard reading nor a default: the application reports none.""" + assert get_type_hints(SnmpValueSpec)["unknown"] is bool + + class Driver: + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = {"cpu": {"unknown": True}} + + assert Driver.SNMP_HEALTH_PROFILE["cpu"].get("unknown") is True + + +def test_the_shapes_live_in_their_own_module_and_models_still_exports_them(): + from napalm_device_types import models, snmp_health_profile + + for name in ("SnmpHealthProfile", "SnmpValueSpec", "SnmpMemorySpec", "SnmpInterfacesSpec"): + assert getattr(models, name) is getattr(snmp_health_profile, name), name + assert getattr(snmp_health_profile, name).__module__ == snmp_health_profile.__name__ + + def test_the_collector_in_the_base_package_warns_that_it_is_deprecated(): async def get(oid): return None diff --git a/tests/test_vm_provision_incomplete.py b/tests/test_vm_provision_incomplete.py new file mode 100644 index 0000000..ddaef8d --- /dev/null +++ b/tests/test_vm_provision_incomplete.py @@ -0,0 +1,53 @@ +"""A provisioning that failed after the hypervisor gave the VM its id says +which VM it left behind, so the caller can remove it (NAPALM/napalm-proxmox#28).""" + +from __future__ import annotations + +import pickle + +import pytest + +from napalm_device_types import VMProvisionIncomplete +from napalm_device_types.provisioning import VMProvisionIncomplete as from_module + + +def _raise_from(cause: Exception) -> VMProvisionIncomplete: + try: + try: + raise cause + except Exception as exc: + raise VMProvisionIncomplete(str(exc), vmid="101", node="pve1") from exc + except VMProvisionIncomplete as caught: + return caught + + +def test_it_is_exported_from_the_package_root(): + assert VMProvisionIncomplete is from_module + + +def test_it_names_the_vm_left_behind(): + caught = _raise_from(RuntimeError("disk import failed")) + assert (caught.vmid, caught.node) == ("101", "pve1") + + +def test_it_keeps_the_failure_and_its_words(): + cause = RuntimeError("disk import failed") + caught = _raise_from(cause) + assert caught.__cause__ is cause + assert str(caught) == "disk import failed" + + +def test_a_caller_catching_runtime_error_still_catches_it(): + # The contract has create_vm_from_cloud_init raise RuntimeError on failure. + with pytest.raises(RuntimeError): + raise VMProvisionIncomplete("start failed", vmid="7", node="n") + + +def test_vmid_and_node_are_keyword_only(): + with pytest.raises(TypeError): + VMProvisionIncomplete("failed", "101", "pve1") # type: ignore[misc] + + +def test_it_survives_pickling_as_a_task_result_would(): + caught = pickle.loads(pickle.dumps(VMProvisionIncomplete("failed", vmid="101", node="pve1"))) + assert (str(caught), caught.vmid, caught.node) == ("failed", "101", "pve1")