From b4cdcfd34704c0fde36902eb0338fba9dcca334d Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 12 Nov 2026 19:07:31 +0100 Subject: [PATCH] fix(dhcp): release an address before another reservation takes it apply_dhcp_reservationset wrote adds first, then updates in the order of the desired set. Kea keeps addresses unique at each write, so when an address moved from one reservation to another, the reservation taking it was refused as a duplicate whenever it came first. The changes are now ordered (_dhcp_order): an update that moves a reservation off its address goes before the change that takes that address, along a chain from its free end, and adds come after every update. A swap or rotation of addresses has no such order; its changes are still tried, so a server that checks only at the commit takes them, and a refused one's [failed] line says it was a swap and with which reservations. Nothing is moved to a temporary address: the contract knows no free one, and a failed second write would leave it there. The diff is computed against one read of the reservations, shared with diff_dhcp_reservations. Refused rows still do not stop the others, the rest is committed and DhcpApplyError follows, as since 4.35.1. Closes #106 --- CHANGELOG.md | 11 +++ README.md | 5 +- docs/STRUCTURE.md | 2 +- napalm_device_types/_dhcp_order.py | 116 ++++++++++++++++++++++ napalm_device_types/dhcp.py | 105 ++++++++++++-------- tests/quality_baseline.json | 1 - tests/test_dhcp_diff_apply.py | 150 +++++++++++++++++++++++++++++ 7 files changed, 347 insertions(+), 43 deletions(-) create mode 100644 napalm_device_types/_dhcp_order.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 81838e5..5aedbd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ recorded here; their history is the git log. ## [Unreleased] +### Fixed +- `apply_dhcp_reservationset` writes its changes so an address is released before it is + taken: an update that moves a reservation off its address goes before the change that + takes that address (along a chain, from its free end), and adds come after every + update. A server that keeps addresses unique at each write, such as Kea, no longer + refuses the second reservation as a duplicate when an address moves from one to + another. A swap -- reservations trading addresses, or a longer rotation -- has no such + order: its changes are still tried, and where the server refuses them the `[failed]` + line says it was a swap and with which reservations. Everything else is still written + and committed, and `DhcpApplyError` follows (#106). + ## [4.37.0] - 2026-11-12 ### Added diff --git a/README.md b/README.md index fee08a6..7852913 100644 --- a/README.md +++ b/README.md @@ -493,7 +493,10 @@ The same split applies to `DhcpServerMixin`: `get_dhcp_reservations`/ `apply_dhcp_reservation`/`commit_dhcp_reservations` are abstract (Kea REST on OPNsense, dnsmasq/odhcpd UCI on OpenWrt), while `diff_dhcp_reservations` and `apply_dhcp_reservationset` are concrete — matching by normalised MAC and the -apply-then-commit orchestration are identical for every DHCP server. +apply-then-commit orchestration are identical for every DHCP server. The apply writes +its changes so an address is released before another reservation takes it (an update +that frees an address first, adds last); a swap of addresses has no such order and is +reported as one where the server refuses it. A new driver (FortiGate, pfSense, …) gets `diff_firewall_rules`/ `apply_firewall_ruleset` for free the moment it implements the three abstract diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index d3d671e..ccad117 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -39,7 +39,7 @@ application SNMP polling, ping sweeps, discovery scoring, guest | **Capability** | Declares only. | `packages`, `services`, `updates`, `nat_vpn`, `mac_acl`, `config_lifecycle`, `interface_filter`, `host_reboot` | | **Template** | A concrete method that runs a fixed sequence of abstract steps. A driver implements the steps, or overrides the whole method when its device does it in one go. All values come from the caller. | the fix routines below | | **Contract types** | Interfaces and data shapes other packages implement or fill. | `CommandChannelMixin`, `CommandResult`, `ByteStream`; the declaration of `ContainerEngineMixin`; the reader mixins' public method declarations; the SNMP profile type; `models` | -| **Reconciler** | Concrete and vendor-neutral, without I/O of its own. | `firewall_rules` and `dhcp` diff/apply, `_matching`, `lag` | +| **Reconciler** | Concrete and vendor-neutral, without I/O of its own. | `firewall_rules` and `dhcp` diff/apply, `_matching`, `_dhcp_order`, `lag` | | **Introspection** | Concrete, without I/O. | `roles`; `base` (`FingerprintRule`, `PortSpec`, the fingerprint attributes) | | **Driver facts** | Class attributes that describe the device or how it is reached, not what an application does with it. | `USES_SSH`, `FULL_UPGRADE`, `platform`, `SNMP_HEALTH_PROFILE` | diff --git a/napalm_device_types/_dhcp_order.py b/napalm_device_types/_dhcp_order.py new file mode 100644 index 0000000..5dbde04 --- /dev/null +++ b/napalm_device_types/_dhcp_order.py @@ -0,0 +1,116 @@ +"""The order an applied reservation set is written in (#106). + +A DHCP server such as Kea keeps every address unique at each write, not only at the +commit: a reservation that takes an address another one still holds is refused as a +duplicate. So the changes are ordered for an address to be released before it is taken: + +* An update that moves a reservation off its address goes before the change that takes + that address -- along a chain, from its free end. +* Adds come after every update: they release nothing, and an update may free their + address. + +A *swap* -- two reservations trading addresses, or a longer rotation -- has no such +order. Its changes are still tried, in the order of the desired set: a server that checks +only at the commit takes them. A server that refuses them refuses each with its own +message, and the progress line says it was a swap. Nothing is moved to a temporary +address: the contract knows no free one (pools, leases and reservations nobody manages +all hold addresses), and a second write that failed would leave the reservation on it. + +Without I/O: a reconciler's part, see ``docs/STRUCTURE.md``. +""" + +from __future__ import annotations + +import ipaddress +from dataclasses import dataclass + +from napalm_device_types.models import DhcpReservationDict, DhcpReservationDiffDict + + +@dataclass(frozen=True) +class ReservationChange: + """One write of an applied set, in the order it is made.""" + + reservation: DhcpReservationDict + #: The live reservation's id for an update, None for an add. + uuid: str | None + #: The changed fields of an update (``"ip, hostname"``), empty for an add. + fields: str = "" + #: The other reservations' MACs of the swap this change is part of, if any. + swap: tuple[str, ...] = () + + +def _address(ip: str | None) -> str: + """*ip* in one spelling, so ``192.0.2.010``-style differences cannot hide a match.""" + text = (ip or "").strip() + try: + return str(ipaddress.ip_address(text)) + except ValueError: + return text.lower() + + +def _waits_on(updates: list[tuple[str, str]]) -> list[int | None]: + """For each ``(old, new)`` address pair, the update whose release it waits for.""" + released = {old: i for i, (old, new) in enumerate(updates) if old and old != new} + return [released.get(new) if new != old else None for old, new in updates] + + +def _cycles(waits_on: list[int | None]) -> dict[int, list[int]]: + """Each update that is part of a cycle, with the cycle's members in order.""" + members: dict[int, list[int]] = {} + for start in range(len(waits_on)): + path: list[int] = [] + node: int | None = start + while node is not None and node not in path and node not in members: + path.append(node) + node = waits_on[node] + if node is not None and node in path: + cycle = sorted(path[path.index(node) :]) + members.update(dict.fromkeys(cycle, cycle)) + return members + + +def _update_order(waits_on: list[int | None], cycles: dict[int, list[int]]) -> list[int]: + """Every update after the one it waits on; stable otherwise. Where only cycles are + left, the first of them goes as it is, and so on.""" + order: list[int] = [] + done: set[int] = set() + pending = list(range(len(waits_on))) + while pending: + ready = [i for i in pending if waits_on[i] is None or waits_on[i] in done] + if not ready: + ready = [next(i for i in pending if i in cycles)] + order += ready + done.update(ready) + pending = [i for i in pending if i not in done] + return order + + +def order_reservation_changes( + diff: DhcpReservationDiffDict, live: list[DhcpReservationDict] +) -> list[ReservationChange]: + """The changes of *diff* in the order they are written; see the module. + + :param diff: What ``diff_dhcp_reservations`` found. + :param live: The reservations it compared against, for the addresses updates release. + """ + held = {r.get("uuid"): _address(r.get("ip")) for r in live if r.get("uuid")} + updates = diff["update"] + waits_on = _waits_on( + [(held.get(e["uuid"], ""), _address(e["reservation"].get("ip"))) for e in updates] + ) + cycles = _cycles(waits_on) + + def swap(i: int) -> tuple[str, ...]: + return tuple(updates[j]["reservation"]["mac"] for j in cycles.get(i, []) if j != i) + + ordered = [ + ReservationChange( + updates[i]["reservation"], + updates[i]["uuid"], + ", ".join(updates[i]["changed_fields"]), + swap(i), + ) + for i in _update_order(waits_on, cycles) + ] + return ordered + [ReservationChange(r, None) for r in diff["add"]] diff --git a/napalm_device_types/dhcp.py b/napalm_device_types/dhcp.py index 12c194c..abb845b 100644 --- a/napalm_device_types/dhcp.py +++ b/napalm_device_types/dhcp.py @@ -26,6 +26,7 @@ import re from collections.abc import Iterator from typing import TYPE_CHECKING, Any +from napalm_device_types._dhcp_order import ReservationChange, order_reservation_changes from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( DhcpHaDict, @@ -57,6 +58,12 @@ _DHCP_SUBNET_COMPARE_FIELDS = ( _HEX_ONLY = re.compile(r"[^0-9a-f]") +#: Added to a refused change that is part of an address swap (#106). +_SWAP_NOTE = ( + "an address swap with {}: no order frees an address first; " + "move one of them to a free address in a run of its own" +) + class DhcpApplyError(RuntimeError): """Some reservations of an applied set were refused by the device. @@ -119,6 +126,43 @@ def _subnet_field_differs(field: str, live: DhcpSubnetDict, desired: DhcpSubnetD return any(live_options.get(option) != value for option, value in desired_options.items()) +def _reservation_key(reservation: DhcpReservationDict) -> str: + return normalize_mac(reservation.get("mac")) + + +def _diff_reservations( + live: list[DhcpReservationDict], desired: list[DhcpReservationDict] +) -> DhcpReservationDiffDict: + """``diff_dhcp_reservations`` against reservations already read.""" + live_by_mac, shared = index_live(live, _reservation_key) + check_desired(desired, _reservation_key, shared, "MAC") + + add: list[DhcpReservationDict] = [] + update: list[DhcpReservationUpdateDict] = [] + + for desired_reservation in desired: + current = live_by_mac.get(_reservation_key(desired_reservation)) + if current is None: + add.append(desired_reservation) + continue + + changed_fields = [ + field + for field in _DHCP_RESERVATION_COMPARE_FIELDS + if current.get(field) != desired_reservation.get(field) + ] + if changed_fields: + update.append( + { + "uuid": current["uuid"], + "reservation": desired_reservation, + "changed_fields": changed_fields, + } + ) + + return {"add": add, "update": update} + + class DhcpServerMixin: """Mixin providing DHCP reservation and subnet read/diff/apply. @@ -271,36 +315,7 @@ class DhcpServerMixin: :raises ValueError: When a desired reservation cannot name a single live one. """ - def key(reservation: DhcpReservationDict) -> str: - return normalize_mac(reservation.get("mac")) - - live_by_mac, shared = index_live(self.get_dhcp_reservations(), key) - check_desired(desired, key, shared, "MAC") - - add: list[DhcpReservationDict] = [] - update: list[DhcpReservationUpdateDict] = [] - - for desired_reservation in desired: - live = live_by_mac.get(key(desired_reservation)) - if live is None: - add.append(desired_reservation) - continue - - changed_fields = [ - field - for field in _DHCP_RESERVATION_COMPARE_FIELDS - if live.get(field) != desired_reservation.get(field) - ] - if changed_fields: - update.append( - { - "uuid": live["uuid"], - "reservation": desired_reservation, - "changed_fields": changed_fields, - } - ) - - return {"add": add, "update": update} + return _diff_reservations(self.get_dhcp_reservations(), desired) def apply_dhcp_reservationset(self, desired: list[DhcpReservationDict]) -> Iterator[str]: """ @@ -311,6 +326,9 @@ class DhcpServerMixin: committing reloads the DHCP daemon and drops in-flight requests, which is too high a price for a no-op run. + The changes are written so an address is released before it is taken; a + swap of addresses has no such order and is reported as one (``_dhcp_order``). + A change the device refuses (say, an address in a subnet its DHCP server does not manage) does not stop the others: it gets a ``[failed]`` line, the rest are written and committed, and ``DhcpApplyError`` is raised at @@ -322,7 +340,8 @@ class DhcpServerMixin: plus a final commit line. :raises DhcpApplyError: after the commit, when a change was refused. """ - diff = self.diff_dhcp_reservations(desired) + live = self.get_dhcp_reservations() + diff = _diff_reservations(live, desired) attempted = len(diff["add"]) + len(diff["update"]) if not attempted: yield "[commit] no changes" @@ -330,17 +349,13 @@ class DhcpServerMixin: added = updated = refused = 0 - for reservation in diff["add"]: - line, ok = self._write_reservation(reservation, None, "[add]") - added, refused = added + ok, refused + (not ok) + for change in order_reservation_changes(diff, live): + line, ok = self._write_change(change) + refused += not ok + added += ok and change.uuid is None + updated += ok and change.uuid is not None yield line - for entry in diff["update"]: - fields = ", ".join(entry["changed_fields"]) - line, ok = self._write_reservation(entry["reservation"], entry["uuid"], "[update]") - updated, refused = updated + ok, refused + (not ok) - yield f"{line} ({fields})" if ok else line - # Committed even when every write was refused: a run that stopped before its # commit left saved writes behind, which the device reports as configured, so # no later diff would ever commit them. @@ -350,6 +365,16 @@ class DhcpServerMixin: if refused: raise DhcpApplyError(refused, attempted) + def _write_change(self, change: ReservationChange) -> tuple[str, bool]: + """Write one change of an ordered set; its progress line, and whether it went in.""" + tag = "[add]" if change.uuid is None else "[update]" + line, ok = self._write_reservation(change.reservation, change.uuid, tag) + if ok and change.fields: + line = f"{line} ({change.fields})" + elif not ok and change.swap: + line = f"{line} -- {_SWAP_NOTE.format(', '.join(change.swap))}" + return line, ok + def _write_reservation( self, reservation: DhcpReservationDict, uuid: str | None, tag: str ) -> tuple[str, bool]: diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json index d75b3d2..4065f83 100644 --- a/tests/quality_baseline.json +++ b/tests/quality_baseline.json @@ -20,7 +20,6 @@ }, "function_lines": { "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 119, - "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_reservations": 54, "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_subnets": 60, "napalm_device_types/firewall_rules.py::FirewallRuleMixin.diff_firewall_rules": 56, "napalm_device_types/hypervisor.py::HypervisorDriver.create_vm_from_cloud_init": 84, diff --git a/tests/test_dhcp_diff_apply.py b/tests/test_dhcp_diff_apply.py index 3440862..35472a4 100644 --- a/tests/test_dhcp_diff_apply.py +++ b/tests/test_dhcp_diff_apply.py @@ -298,6 +298,156 @@ class TestApplyWithARefusedReservation: assert issubclass(DhcpApplyError, RuntimeError) +#: What Kea answers through OPNsense's API for an address another reservation holds. +_KEA_DUPLICATE = ( + "{'result': 'failed', 'validations': {'reservation.ip_address': 'Duplicate entry exists.'}}" +) + + +class _KeaLikeServer(_FakeDhcpServer): + """A server that keeps each address unique at every write, as Kea does: a write + taking an address another reservation still holds is refused.""" + + def apply_dhcp_reservation( + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: + if any(r["ip"] == reservation["ip"] and r["uuid"] != uuid for r in self.live): + raise RuntimeError(_KEA_DUPLICATE) + stored = {**reservation, "uuid": uuid or f"new-{len(self.live)}"} + self.live = [r for r in self.live if r["uuid"] != uuid] + [stored] # type: ignore[list-item] + return super().apply_dhcp_reservation(reservation, uuid=uuid) + + def ip_of(self, mac: str) -> str: + return next(r["ip"] for r in self.live if r["mac"] == mac) + + +_A, _B, _C, _N = (f"02:00:00:00:00:0{i}" for i in "abcd") + + +def _at(mac: str, ip: str, uuid: str = "") -> DhcpReservationDict: + return _reservation(mac=mac, ip=ip, uuid=uuid, subnet="192.0.2.0/24") + + +class TestApplyOrderReleasesBeforeTaking: + """An address is released before another change takes it (#106).""" + + def test_an_update_that_frees_an_address_goes_first(self): + # The issue: B takes A's address, A moves on; desired lists B first. + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua"), _at(_B, "192.0.2.11", "ub")]) + + lines = list( + driver.apply_dhcp_reservationset([_at(_B, "192.0.2.10"), _at(_A, "192.0.2.20")]) + ) + + assert driver.ip_of(_A) == "192.0.2.20" + assert driver.ip_of(_B) == "192.0.2.10" + assert lines == [ + f"[update] {_A} -> 192.0.2.20 (ip)", + f"[update] {_B} -> 192.0.2.10 (ip)", + "[commit] applied 0 add(s), 2 update(s)", + ] + + def test_a_chain_is_written_from_its_free_end(self): + live = [ + _at(_A, "192.0.2.10", "ua"), + _at(_B, "192.0.2.11", "ub"), + _at(_C, "192.0.2.12", "uc"), + ] + driver = _KeaLikeServer(live) + + list( + driver.apply_dhcp_reservationset( + [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.12"), _at(_C, "192.0.2.13")] + ) + ) + + assert [e["reservation"]["mac"] for e in driver.applied] == [_C, _B, _A] + assert [driver.ip_of(m) for m in (_A, _B, _C)] == ["192.0.2.11", "192.0.2.12", "192.0.2.13"] + + def test_an_add_comes_after_the_update_that_frees_its_address(self): + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua")]) + + lines = list( + driver.apply_dhcp_reservationset([_at(_N, "192.0.2.10"), _at(_A, "192.0.2.20")]) + ) + + assert driver.ip_of(_N) == "192.0.2.10" + assert driver.ip_of(_A) == "192.0.2.20" + assert lines[-1] == "[commit] applied 1 add(s), 1 update(s)" + + def test_addresses_match_however_they_are_spelled(self): + driver = _FakeDhcpServer([_at(_A, "2001:DB8::10", "ua"), _at(_B, "2001:DB8::11", "ub")]) + + list(driver.apply_dhcp_reservationset([_at(_B, "2001:db8::10"), _at(_A, "2001:db8::20")])) + + assert [e["reservation"]["mac"] for e in driver.applied] == [_A, _B] + + def test_an_update_that_keeps_its_address_frees_nothing(self): + # A hostname change is no release: the add for A's address stays refused. + driver = _KeaLikeServer([_at(_A, "192.0.2.10", "ua")]) + + with pytest.raises(DhcpApplyError, match="1 of 2"): + list( + driver.apply_dhcp_reservationset( + [_at(_N, "192.0.2.10"), {**_at(_A, "192.0.2.10"), "hostname": "renamed"}] + ) + ) + + assert driver.ip_of(_A) == "192.0.2.10" + + +class TestApplyOrderWithASwap: + """Two reservations trading addresses: no order frees one first. Each is still + tried -- a server that checks only at the commit takes them -- and a refusal says it + was a swap. Nothing is moved to a temporary address: the contract has no way to know + a free one, and a second write that failed would leave the reservation there.""" + + def _swap(self) -> list[DhcpReservationDict]: + return [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.10"), _at(_N, "192.0.2.30")] + + def _live(self) -> list[DhcpReservationDict]: + return [_at(_A, "192.0.2.10", "ua"), _at(_B, "192.0.2.11", "ub")] + + def test_a_refused_swap_is_reported_as_one_and_the_rest_is_written(self): + driver = _KeaLikeServer(self._live()) + lines: list[str] = [] + + with pytest.raises(DhcpApplyError, match="2 of 3"): + for line in driver.apply_dhcp_reservationset(self._swap()): + lines.append(line) + + failed = [line for line in lines if line.startswith("[failed]")] + assert len(failed) == 2 + assert all("Duplicate entry exists" in line for line in failed) + assert f"swap with {_B}" in failed[0] and f"swap with {_A}" in failed[1] + assert driver.ip_of(_A) == "192.0.2.10" and driver.ip_of(_B) == "192.0.2.11" + assert driver.ip_of(_N) == "192.0.2.30" + assert driver.commits == 1 + + def test_a_server_that_takes_the_swap_gets_it(self): + driver = _FakeDhcpServer(self._live()) + + lines = list(driver.apply_dhcp_reservationset(self._swap())) + + assert len(driver.applied) == 3 + assert not any("swap" in line for line in lines) + + def test_a_rotation_of_three_is_a_swap_too(self): + driver = _KeaLikeServer([*self._live(), _at(_C, "192.0.2.12", "uc")]) + lines: list[str] = [] + + with pytest.raises(DhcpApplyError, match="3 of 3"): + for line in driver.apply_dhcp_reservationset( + [_at(_A, "192.0.2.11"), _at(_B, "192.0.2.12"), _at(_C, "192.0.2.10")] + ): + lines.append(line) + + by_mac = {line.split()[1]: line for line in lines if line.startswith("[failed]")} + assert f"swap with {_B}, {_C}" in by_mac[_A] + assert f"swap with {_A}, {_C}" in by_mac[_B] + assert f"swap with {_A}, {_B}" in by_mac[_C] + + class TestKeysThatCannotNameOneReservation: def test_a_live_reservation_without_mac_is_never_matched(self): driver = _FakeDhcpServer([_reservation(uuid="b", mac="")]) -- 2.54.0