diff --git a/CHANGELOG.md b/CHANGELOG.md index 81838e5..5aedbd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ recorded here; their history is the git log. ## [Unreleased] +### Fixed +- `apply_dhcp_reservationset` writes its changes so an address is released before it is + taken: an update that moves a reservation off its address goes before the change that + takes that address (along a chain, from its free end), and adds come after every + update. A server that keeps addresses unique at each write, such as Kea, no longer + refuses the second reservation as a duplicate when an address moves from one to + another. A swap -- reservations trading addresses, or a longer rotation -- has no such + order: its changes are still tried, and where the server refuses them the `[failed]` + line says it was a swap and with which reservations. Everything else is still written + and committed, and `DhcpApplyError` follows (#106). + ## [4.37.0] - 2026-11-12 ### Added diff --git a/README.md b/README.md index fee08a6..7852913 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,10 @@ The same split applies to `DhcpServerMixin`: `get_dhcp_reservations`/ `apply_dhcp_reservation`/`commit_dhcp_reservations` are abstract (Kea REST on OPNsense, dnsmasq/odhcpd UCI on OpenWrt), while `diff_dhcp_reservations` and `apply_dhcp_reservationset` are concrete — matching by normalised MAC and the -apply-then-commit orchestration are identical for every DHCP server. +apply-then-commit orchestration are identical for every DHCP server. The apply writes +its changes so an address is released before another reservation takes it (an update +that frees an address first, adds last); a swap of addresses has no such order and is +reported as one where the server refuses it. A new driver (FortiGate, pfSense, …) gets `diff_firewall_rules`/ `apply_firewall_ruleset` for free the moment it implements the three abstract diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index d3d671e..ccad117 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -39,7 +39,7 @@ application SNMP polling, ping sweeps, discovery scoring, guest | **Capability** | Declares only. | `packages`, `services`, `updates`, `nat_vpn`, `mac_acl`, `config_lifecycle`, `interface_filter`, `host_reboot` | | **Template** | A concrete method that runs a fixed sequence of abstract steps. A driver implements the steps, or overrides the whole method when its device does it in one go. All values come from the caller. | the fix routines below | | **Contract types** | Interfaces and data shapes other packages implement or fill. | `CommandChannelMixin`, `CommandResult`, `ByteStream`; the declaration of `ContainerEngineMixin`; the reader mixins' public method declarations; the SNMP profile type; `models` | -| **Reconciler** | Concrete and vendor-neutral, without I/O of its own. | `firewall_rules` and `dhcp` diff/apply, `_matching`, `lag` | +| **Reconciler** | Concrete and vendor-neutral, without I/O of its own. | `firewall_rules` and `dhcp` diff/apply, `_matching`, `_dhcp_order`, `lag` | | **Introspection** | Concrete, without I/O. | `roles`; `base` (`FingerprintRule`, `PortSpec`, the fingerprint attributes) | | **Driver facts** | Class attributes that describe the device or how it is reached, not what an application does with it. | `USES_SSH`, `FULL_UPGRADE`, `platform`, `SNMP_HEALTH_PROFILE` | diff --git a/napalm_device_types/_dhcp_order.py b/napalm_device_types/_dhcp_order.py new file mode 100644 index 0000000..5dbde04 --- /dev/null +++ b/napalm_device_types/_dhcp_order.py @@ -0,0 +1,116 @@ +"""The order an applied reservation set is written in (#106). + +A DHCP server such as Kea keeps every address unique at each write, not only at the +commit: a reservation that takes an address another one still holds is refused as a +duplicate. So the changes are ordered for an address to be released before it is taken: + +* An update that moves a reservation off its address goes before the change that takes + that address -- along a chain, from its free end. +* Adds come after every update: they release nothing, and an update may free their + address. + +A *swap* -- two reservations trading addresses, or a longer rotation -- has no such +order. Its changes are still tried, in the order of the desired set: a server that checks +only at the commit takes them. A server that refuses them refuses each with its own +message, and the progress line says it was a swap. Nothing is moved to a temporary +address: the contract knows no free one (pools, leases and reservations nobody manages +all hold addresses), and a second write that failed would leave the reservation on it. + +Without I/O: a reconciler's part, see ``docs/STRUCTURE.md``. +""" + +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass + +from napalm_device_types.models import DhcpReservationDict, DhcpReservationDiffDict + + +@dataclass(frozen=True) +class ReservationChange: + """One write of an applied set, in the order it is made.""" + + reservation: DhcpReservationDict + #: The live reservation's id for an update, None for an add. + uuid: str | None + #: The changed fields of an update (``"ip, hostname"``), empty for an add. + fields: str = "" + #: The other reservations' MACs of the swap this change is part of, if any. + swap: tuple[str, ...] = () + + +def _address(ip: str | None) -> str: + """*ip* in one spelling, so ``192.0.2.010``-style differences cannot hide a match.""" + text = (ip or "").strip() + try: + return str(ipaddress.ip_address(text)) + except ValueError: + return text.lower() + + +def _waits_on(updates: list[tuple[str, str]]) -> list[int | None]: + """For each ``(old, new)`` address pair, the update whose release it waits for.""" + released = {old: i for i, (old, new) in enumerate(updates) if old and old != new} + return [released.get(new) if new != old else None for old, new in updates] + + +def _cycles(waits_on: list[int | None]) -> dict[int, list[int]]: + """Each update that is part of a cycle, with the cycle's members in order.""" + members: dict[int, list[int]] = {} + for start in range(len(waits_on)): + path: list[int] = [] + node: int | None = start + while node is not None and node not in path and node not in members: + path.append(node) + node = waits_on[node] + if node is not None and node in path: + cycle = sorted(path[path.index(node) :]) + members.update(dict.fromkeys(cycle, cycle)) + return members + + +def _update_order(waits_on: list[int | None], cycles: dict[int, list[int]]) -> list[int]: + """Every update after the one it waits on; stable otherwise. Where only cycles are + left, the first of them goes as it is, and so on.""" + order: list[int] = [] + done: set[int] = set() + pending = list(range(len(waits_on))) + while pending: + ready = [i for i in pending if waits_on[i] is None or waits_on[i] in done] + if not ready: + ready = [next(i for i in pending if i in cycles)] + order += ready + done.update(ready) + pending = [i for i in pending if i not in done] + return order + + +def order_reservation_changes( + diff: DhcpReservationDiffDict, live: list[DhcpReservationDict] +) -> list[ReservationChange]: + """The changes of *diff* in the order they are written; see the module. + + :param diff: What ``diff_dhcp_reservations`` found. + :param live: The reservations it compared against, for the addresses updates release. + """ + held = {r.get("uuid"): _address(r.get("ip")) for r in live if r.get("uuid")} + updates = diff["update"] + waits_on = _waits_on( + [(held.get(e["uuid"], ""), _address(e["reservation"].get("ip"))) for e in updates] + ) + cycles = _cycles(waits_on) + + def swap(i: int) -> tuple[str, ...]: + return tuple(updates[j]["reservation"]["mac"] for j in cycles.get(i, []) if j != i) + + ordered = [ + ReservationChange( + updates[i]["reservation"], + updates[i]["uuid"], + ", ".join(updates[i]["changed_fields"]), + swap(i), + ) + for i in _update_order(waits_on, cycles) + ] + return ordered + [ReservationChange(r, None) for r in diff["add"]] diff --git a/napalm_device_types/dhcp.py b/napalm_device_types/dhcp.py index 12c194c..abb845b 100644 --- a/napalm_device_types/dhcp.py +++ b/napalm_device_types/dhcp.py @@ -26,6 +26,7 @@ import re from collections.abc import Iterator from typing import TYPE_CHECKING, Any +from napalm_device_types._dhcp_order import ReservationChange, order_reservation_changes from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( DhcpHaDict, @@ -57,6 +58,12 @@ _DHCP_SUBNET_COMPARE_FIELDS = ( _HEX_ONLY = re.compile(r"[^0-9a-f]") +#: Added to a refused change that is part of an address swap (#106). +_SWAP_NOTE = ( + "an address swap with {}: no order frees an address first; " + "move one of them to a free address in a run of its own" +) + class DhcpApplyError(RuntimeError): """Some reservations of an applied set were refused by the device. @@ -119,6 +126,43 @@ def _subnet_field_differs(field: str, live: DhcpSubnetDict, desired: DhcpSubnetD return any(live_options.get(option) != value for option, value in desired_options.items()) +def _reservation_key(reservation: DhcpReservationDict) -> str: + return normalize_mac(reservation.get("mac")) + + +def _diff_reservations( + live: list[DhcpReservationDict], desired: list[DhcpReservationDict] +) -> DhcpReservationDiffDict: + """``diff_dhcp_reservations`` against reservations already read.""" + live_by_mac, shared = index_live(live, _reservation_key) + check_desired(desired, _reservation_key, shared, "MAC") + + add: list[DhcpReservationDict] = [] + update: list[DhcpReservationUpdateDict] = [] + + for desired_reservation in desired: + current = live_by_mac.get(_reservation_key(desired_reservation)) + if current is None: + add.append(desired_reservation) + continue + + changed_fields = [ + field + for field in _DHCP_RESERVATION_COMPARE_FIELDS + if current.get(field) != desired_reservation.get(field) + ] + if changed_fields: + update.append( + { + "uuid": current["uuid"], + "reservation": desired_reservation, + "changed_fields": changed_fields, + } + ) + + return {"add": add, "update": update} + + class DhcpServerMixin: """Mixin providing DHCP reservation and subnet read/diff/apply. @@ -271,36 +315,7 @@ class DhcpServerMixin: :raises ValueError: When a desired reservation cannot name a single live one. """ - def key(reservation: DhcpReservationDict) -> str: - return normalize_mac(reservation.get("mac")) - - live_by_mac, shared = index_live(self.get_dhcp_reservations(), key) - check_desired(desired, key, shared, "MAC") - - add: list[DhcpReservationDict] = [] - update: list[DhcpReservationUpdateDict] = [] - - for desired_reservation in desired: - live = live_by_mac.get(key(desired_reservation)) - if live is None: - add.append(desired_reservation) - continue - - changed_fields = [ - field - for field in _DHCP_RESERVATION_COMPARE_FIELDS - if live.get(field) != desired_reservation.get(field) - ] - if changed_fields: - update.append( - { - "uuid": live["uuid"], - "reservation": desired_reservation, - "changed_fields": changed_fields, - } - ) - - return {"add": add, "update": update} + return _diff_reservations(self.get_dhcp_reservations(), desired) def apply_dhcp_reservationset(self, desired: list[DhcpReservationDict]) -> Iterator[str]: """ @@ -311,6 +326,9 @@ class DhcpServerMixin: committing reloads the DHCP daemon and drops in-flight requests, which is too high a price for a no-op run. + The changes are written so an address is released before it is taken; a + swap of addresses has no such order and is reported as one (``_dhcp_order``). + A change the device refuses (say, an address in a subnet its DHCP server does not manage) does not stop the others: it gets a ``[failed]`` line, the rest are written and committed, and ``DhcpApplyError`` is raised at @@ -322,7 +340,8 @@ class DhcpServerMixin: plus a final commit line. :raises DhcpApplyError: after the commit, when a change was refused. """ - diff = self.diff_dhcp_reservations(desired) + live = self.get_dhcp_reservations() + diff = _diff_reservations(live, desired) attempted = len(diff["add"]) + len(diff["update"]) if not attempted: yield "[commit] no changes" @@ -330,17 +349,13 @@ class DhcpServerMixin: added = updated = refused = 0 - for reservation in diff["add"]: - line, ok = self._write_reservation(reservation, None, "[add]") - added, refused = added + ok, refused + (not ok) + for change in order_reservation_changes(diff, live): + line, ok = self._write_change(change) + refused += not ok + added += ok and change.uuid is None + updated += ok and change.uuid is not None yield line - for entry in diff["update"]: - fields = ", ".join(entry["changed_fields"]) - line, ok = self._write_reservation(entry["reservation"], entry["uuid"], "[update]") - updated, refused = updated + ok, refused + (not ok) - yield f"{line} ({fields})" if ok else line - # Committed even when every write was refused: a run that stopped before its # commit left saved writes behind, which the device reports as configured, so # no later diff would ever commit them. @@ -350,6 +365,16 @@ class DhcpServerMixin: if refused: raise DhcpApplyError(refused, attempted) + def _write_change(self, change: ReservationChange) -> tuple[str, bool]: + """Write one change of an ordered set; its progress line, and whether it went in.""" + tag = "[add]" if change.uuid is None else "[update]" + line, ok = self._write_reservation(change.reservation, change.uuid, tag) + if ok and change.fields: + line = f"{line} ({change.fields})" + elif not ok and change.swap: + line = f"{line} -- {_SWAP_NOTE.format(', '.join(change.swap))}" + return line, ok + def _write_reservation( self, reservation: DhcpReservationDict, uuid: str | None, tag: str ) -> tuple[str, bool]: diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index d75b3d2..4065f83 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -20,7 +20,6 @@ }, "function_lines": { "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 119, - "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_reservations": 54, "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_subnets": 60, "napalm_device_types/firewall_rules.py::FirewallRuleMixin.diff_firewall_rules": 56, "napalm_device_types/hypervisor.py::HypervisorDriver.create_vm_from_cloud_init": 84, diff --git a/tests/test_dhcp_diff_apply.py b/tests/test_dhcp_diff_apply.py index 3440862..35472a4 100644 --- a/tests/test_dhcp_diff_apply.py +++ b/tests/test_dhcp_diff_apply.py @@ -298,6 +298,156 @@ class TestApplyWithARefusedReservation: assert issubclass(DhcpApplyError, RuntimeError) +#: What Kea answers through OPNsense's API for an address another reservation holds. +_KEA_DUPLICATE = ( + "{'result': 'failed', 'validations': {'reservation.ip_address': 'Duplicate entry exists.'}}" +) + + +class _KeaLikeServer(_FakeDhcpServer): + """A server that keeps each address unique at every write, as Kea does: a write + taking an address another reservation still holds is refused.""" + + def apply_dhcp_reservation( + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: + if any(r["ip"] == reservation["ip"] and r["uuid"] != uuid for r in self.live): + raise RuntimeError(_KEA_DUPLICATE) + stored = {**reservation, "uuid": uuid or f"new-{len(self.live)}"} + self.live = [r for r in self.live if r["uuid"] != uuid] + [stored] # type: ignore[list-item] + return super().apply_dhcp_reservation(reservation, uuid=uuid) + + def ip_of(self, mac: str) -> str: + return next(r["ip"] for r in self.live if r["mac"] == mac) + + +_A, _B, _C, _N = (f"02:00:00:00:00:0{i}" for i in "abcd") + + +def _at(mac: str, ip: str, uuid: str = "") -> DhcpReservationDict: + return _reservation(mac=mac, ip=ip, uuid=uuid, subnet="192.0.2.0/24") + + +class TestApplyOrderReleasesBeforeTaking: + """An address is released before another change takes it (#106).""" + + def test_an_update_that_frees_an_address_goes_first(self): + # The issue: B takes A's address, A moves on; desired lists B first. + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua"), _at(_B, "192.0.2.11", "ub")]) + + lines = list( + driver.apply_dhcp_reservationset([_at(_B, "192.0.2.10"), _at(_A, "192.0.2.20")]) + ) + + assert driver.ip_of(_A) == "192.0.2.20" + assert driver.ip_of(_B) == "192.0.2.10" + assert lines == [ + f"[update] {_A} -> 192.0.2.20 (ip)", + f"[update] {_B} -> 192.0.2.10 (ip)", + "[commit] applied 0 add(s), 2 update(s)", + ] + + def test_a_chain_is_written_from_its_free_end(self): + live = [ + _at(_A, "192.0.2.10", "ua"), + _at(_B, "192.0.2.11", "ub"), + _at(_C, "192.0.2.12", "uc"), + ] + driver = _KeaLikeServer(live) + + list( + driver.apply_dhcp_reservationset( + [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.12"), _at(_C, "192.0.2.13")] + ) + ) + + assert [e["reservation"]["mac"] for e in driver.applied] == [_C, _B, _A] + assert [driver.ip_of(m) for m in (_A, _B, _C)] == ["192.0.2.11", "192.0.2.12", "192.0.2.13"] + + def test_an_add_comes_after_the_update_that_frees_its_address(self): + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua")]) + + lines = list( + driver.apply_dhcp_reservationset([_at(_N, "192.0.2.10"), _at(_A, "192.0.2.20")]) + ) + + assert driver.ip_of(_N) == "192.0.2.10" + assert driver.ip_of(_A) == "192.0.2.20" + assert lines[-1] == "[commit] applied 1 add(s), 1 update(s)" + + def test_addresses_match_however_they_are_spelled(self): + driver = _FakeDhcpServer([_at(_A, "2001:DB8::10", "ua"), _at(_B, "2001:DB8::11", "ub")]) + + list(driver.apply_dhcp_reservationset([_at(_B, "2001:db8::10"), _at(_A, "2001:db8::20")])) + + assert [e["reservation"]["mac"] for e in driver.applied] == [_A, _B] + + def test_an_update_that_keeps_its_address_frees_nothing(self): + # A hostname change is no release: the add for A's address stays refused. + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua")]) + + with pytest.raises(DhcpApplyError, match="1 of 2"): + list( + driver.apply_dhcp_reservationset( + [_at(_N, "192.0.2.10"), {**_at(_A, "192.0.2.10"), "hostname": "renamed"}] + ) + ) + + assert driver.ip_of(_A) == "192.0.2.10" + + +class TestApplyOrderWithASwap: + """Two reservations trading addresses: no order frees one first. Each is still + tried -- a server that checks only at the commit takes them -- and a refusal says it + was a swap. Nothing is moved to a temporary address: the contract has no way to know + a free one, and a second write that failed would leave the reservation there.""" + + def _swap(self) -> list[DhcpReservationDict]: + return [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.10"), _at(_N, "192.0.2.30")] + + def _live(self) -> list[DhcpReservationDict]: + return [_at(_A, "192.0.2.10", "ua"), _at(_B, "192.0.2.11", "ub")] + + def test_a_refused_swap_is_reported_as_one_and_the_rest_is_written(self): + driver = _KeaLikeServer(self._live()) + lines: list[str] = [] + + with pytest.raises(DhcpApplyError, match="2 of 3"): + for line in driver.apply_dhcp_reservationset(self._swap()): + lines.append(line) + + failed = [line for line in lines if line.startswith("[failed]")] + assert len(failed) == 2 + assert all("Duplicate entry exists" in line for line in failed) + assert f"swap with {_B}" in failed[0] and f"swap with {_A}" in failed[1] + assert driver.ip_of(_A) == "192.0.2.10" and driver.ip_of(_B) == "192.0.2.11" + assert driver.ip_of(_N) == "192.0.2.30" + assert driver.commits == 1 + + def test_a_server_that_takes_the_swap_gets_it(self): + driver = _FakeDhcpServer(self._live()) + + lines = list(driver.apply_dhcp_reservationset(self._swap())) + + assert len(driver.applied) == 3 + assert not any("swap" in line for line in lines) + + def test_a_rotation_of_three_is_a_swap_too(self): + driver = _KeaLikeServer([*self._live(), _at(_C, "192.0.2.12", "uc")]) + lines: list[str] = [] + + with pytest.raises(DhcpApplyError, match="3 of 3"): + for line in driver.apply_dhcp_reservationset( + [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.12"), _at(_C, "192.0.2.10")] + ): + lines.append(line) + + by_mac = {line.split()[1]: line for line in lines if line.startswith("[failed]")} + assert f"swap with {_B}, {_C}" in by_mac[_A] + assert f"swap with {_A}, {_C}" in by_mac[_B] + assert f"swap with {_A}, {_B}" in by_mac[_C] + + class TestKeysThatCannotNameOneReservation: def test_a_live_reservation_without_mac_is_never_matched(self): driver = _FakeDhcpServer([_reservation(uuid="b", mac="")])