diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..7530c79 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,15 @@ +name: CI + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + +jobs: + driver: + uses: NAPALM/driver-standard/.gitea/workflows/driver.yml@main + with: + standard-ref: main + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 372e184..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: CI - -on: - push: - branches: ["**"] - pull_request: - branches: ["**"] - -jobs: - test: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: pip - - - name: Install package with dev extras - run: | - python -m pip install --upgrade pip - python -m pip install -e ".[dev]" - - - name: Type-check - run: | - python -m mypy napalm_device_types - - - name: Run unit tests - run: | - python -m pytest -q --tb=short - - - name: Build wheel and sdist - run: | - python -m pip install build - python -m build - - - name: Upload dist artifacts - # v4 refuses to run on Gitea ("not currently supported on GHES"). - uses: actions/upload-artifact@v3 - with: - name: dist-${{ matrix.python-version }} - path: dist/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..aa6e349 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,194 @@ +# Changelog + +All notable changes to napalm-device-types. The format follows +[Keep a Changelog](https://keepachangelog.com/), versions follow +[Semantic Versioning](https://semver.org/). Versions up to 4.21.0 were not +recorded here; their history is the git log. + +## [Unreleased] + +### Fixed +- `DNF_SECURITY_COMMAND` reads the cached metadata only (`dnf --cacheonly`), as + `APT_UPGRADABLE_COMMAND` reads apt's local index (#76). With an expired cache dnf + downloaded the repositories' metadata first, and on an Oracle Linux host on OCI that + outlasted netOrk's whole poll budget. Refreshing the index stays a step of its own + (`dnf makecache`). Without a cache dnf fails at once, which already reads as + "security unknown". + +## [4.23.0] - 2026-11-02 + +### Changed +- Python 3.12 is the floor (`requires-python = ">=3.12"`): netOrk runs every driver + on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). +- The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python + 3.12 venv no longer has. +- `requests` is a declared dependency (floor 2.25.0), for `tls_certificates`; it came + with napalm before. +- The SNMP health profile shapes (`SnmpHealthProfile`, `SnmpValueSpec`, + `SnmpMemorySpec`, `SnmpInterfacesSpec`) are defined in + `napalm_device_types.snmp_health_profile`. The package and `models` still export + them, so an existing import keeps working. + +### Added +- `HypervisorDriver.get_vm_migration_preflight(name)`, `migrate_vm(name, target_node, + online=True, target_storage=None, timeout=3600)` and `set_node_maintenance(node, + enabled)`: which nodes of its cluster a guest could move to and what blocks it + (`MigrationPreflightDict`: `allowed_nodes`, `refused_nodes` with the platform's + reason, `local_disks` the migration copies, `local_resources` that block it), the + move itself, blocking until the platform's task ended and raising with the + platform's own cause, and a node's HA maintenance mode. Declared only; netOrk + probes each with `hasattr`. +- `get_clusters()` may report HA, backups and quorum votes: `ClusterDict.ha_resources` + (`HAResourceDict`: requested and current state, node, PVE 8 group, PVE 9 rules), + `backup_jobs` (`BackupJobDict`: schedule, storage, mode, the guests a job covers) + and `backups` (`BackupDict`: per guest and storage the newest backup, its size + and how many there are), and `ClusterNodeDict.votes`. All `NotRequired`, so + drivers written before stay valid; None means could not be read, `[]` read and + empty. +- `napalm_device_types.tls_certificates`: TLS certificates by trust on first use, the + counterpart of `ssh_host_keys` for a driver's HTTPS API. A driver takes the + fingerprint to expect from `optional_args["tls_fingerprints"]` (by port, the SHA-256 + of the DER certificate, `SHA256:AB:CD:…`) and refuses any other certificate with + `TlsCertificateMismatch` (a `ConnectionException` naming both fingerprints) before a + password is sent; no CA is consulted, so a self-signed certificate verifies. Without + a fingerprint the first certificate is accepted. Every certificate seen goes to + `optional_args["tls_certificate_observer"]`. `CertificateCheck.verify_server()` + checks the certificate in a handshake of its own, `pin(session)` makes every later + connection of a `requests` session present the same one, `refusing()` turns a refusal + there into the mismatch, `observe()` reports it once the login stands + (NAPALM/napalm-hpe-aruba-procurve#17). +- `LldpAgentMixin.configure_lldp_agent(interfaces)` (`napalm_device_types.fixes`): run + an LLDP agent on the interfaces the caller names. The template installs the agent + (`ensure_package(LLDP_AGENT_PACKAGE)`), has the driver write the interface list + (`_write_lldp_agent_config`), then enables and restarts the service + (`manage_service(LLDP_AGENT_SERVICE, …)`); both names default to `lldpd`. No + interface is a ValueError before anything changes. It is the action for what LLDP + getters used to do on their own (NAPALM/napalm-openwrt#8). +- `SnmpValueSpec.unknown`: a driver whose device's agent answers a value wrongly says + so in its `SNMP_HEALTH_PROFILE` (`"cpu": {"unknown": True}`), and the application + reports no value for it -- not the standard reading, not 0. OpenBSD's net-snmp, + which reports the CPU as never idle, is the case (NAPALM/napalm-bsd#7). +- `VMProvisionIncomplete` (`napalm_device_types.provisioning`): what + `create_vm_from_cloud_init` raises when it fails after the VM was created. A + `RuntimeError` with the failure as its cause and the half-built VM's `vmid` and + `node`, so the caller can remove it with `destroy_vm` (NAPALM/napalm-proxmox#28). +- `HypervisorDriver.get_clusters()` and its shapes `ClusterDict`, `ClusterNodeDict`, + `ClusterStorageDict`, `ClusterNetworkDict`: every cluster the connection can see, + with its nodes (offline ones included, `local` on the node the connection is), + their CPU and memory, and the storages and networks with the nodes that see each. + A standalone host is a cluster of one with no name; a manager reports each cluster + and standalone host it runs. Declared only (NetOrk/netork#484, #1821). +- `create_vm_from_cloud_init(node=…)`: the cluster node to create the VM on. None + leaves it to the driver; a driver that can only create on its own node raises + ValueError for another (NetOrk/netork#1822). +- `LocalAccountsMixin.add_service_account(name, public_key)`: netOrk's own account, + created through the account the driver logs in with today. It takes a key and no + password, and a rule of its own in `85-service-` lets it run everything as + root without a password (`NOPASSWD: ALL`, checked with `visudo` and `sudo -l`). An + account that is there is opened again and keeps its other keys; root is refused + (NetOrk/netork#1817). With `uid=`, a new account gets that uid once no other account + has it; one that is there keeps its own. A driver supplies + `_account_create_with_uid`. An account that is there and cannot log in (`nologin`, + `false`) is another service's, such as a netOrk satellite's `netork`, and is refused + before anything changes. +- `LocalAccountsMixin` (`napalm_device_types.local_accounts`): personal login accounts + on a host. `get_local_accounts(names)` reads them back (groups, lock, sudo, key + files), `add_local_account(name, public_key, password_hash=, admin=)` creates or + reopens one, `lock_local_account(name)` takes its keys and sudo away and refuses + every login while keeping its home. Every command root runs is one fixed command, + the key files are read and written as the account itself, and the sudo rule is a + drop-in of its own, checked with `visudo` and `sudo -l`. A driver supplies the + commands that create, open and lock an account and take it out of a group, and may + replace where the keys are kept (`_account_key_files`, `_account_read_keys`, + `_account_add_key`, `_account_remove_keys`; default `~/.ssh/authorized_keys`) + (NetOrk/netork#1793). +- `OpenSshServerMixin.exempt_from_ssh_penalties`: puts the address the host sees + the managing client at (`$SSH_CONNECTION`) into `PerSourcePenaltyExemptList`, + in `/etc/ssh/sshd_config.d/00-netork-penalty-exempt.conf`, next to the host's + own exemptions. Only where sshd has per-source penalties (OpenSSH 9.8+; an older + one refuses the directive) and sshd_config includes the drop-in directory; + checked with `sshd -t` and `sshd -T`, then a new session, and taken back on any + failure. Helpers: `parse_penalty_exemptions`, `client_address`, `is_exempt` + (NetOrk/netork#1800). +- `retry_refused` and `refused_before_banner` (`napalm_device_types.ssh_host_keys`): + a connection sshd closes before its banner -- what OpenSSH 9.8+ does to a source + it has just penalised -- is tried again after 1, 3, 7 and 15 s. + `HostKeyCheck.connect` does so itself (NetOrk/netork#1800). +- `ARP_AGE_UNKNOWN` (`napalm_device_types.arp`, `-1.0`): the `age` a driver reports + in `get_arp_table` and `get_ipv6_neighbors_table` when the device does not say how + old an entry is. `0.0` means "confirmed just now" and is no stand-in for it + (NetOrk/netork#1327). +- `NetworkTargetDict.uplinks` (optional): the physical NICs a bridge leaves the + host through, one group per way out -- a bond's slaves together, a single NIC + alone (`[["eno1", "eno2"]]`). A missing key means unknown, an empty list a + bridge without uplink. NetOrk follows their LLDP neighbours to the switch port + and its untagged VLAN; one slave's neighbour speaks for its bond + (NetOrk/netork#1325). +- `HostShutdownMixin.shutdown_host()`, on `DeviceTypeDriver`: shut the device itself + down cleanly through its own means; declared, so `hasattr` says whether a driver + can. No forced variant (NetOrk/netork#1636). +- `VmStopRequestMixin.begin_stop_vm(name)`: ask for a VM's clean shutdown and return + at once, for callers that wait on a timer instead of in `stop_vm`. No role demands + it (NetOrk/netork#1636). +- `SudoHistoryMixin` (`napalm_device_types.sudo_history`): the sudo password an earlier + driver typed into an interactive shell (`echo '' | sudo -S -p "" …`) and + left in the login user's shell history, for a driver to mix in (NetOrk/netork#1090). + The driver says what its lines carry (`SUDO_HISTORY_MARKER`) and which files its + partial sudo left to the login user (`SUDO_ROOT_FILES`), and supplies `run_command`. + - `get_history_sudo_lines()` counts the lines per history file (`~/.bash_history`, + `~/.zsh_history`, `~/.ash_history`, `~/.sh_history`) with `grep -c` as the login + user. No line is read back; a file it cannot read raises. + - `remove_history_sudo_lines()` deletes exactly those lines as the login user (`sed + -i`, its backup overwritten with `shred -u`), and as root gives each of + `SUDO_ROOT_FILES` back to `root:root` where it is a regular file, not a symbolic + link, owned by the login user. It reports every file's lines before and after, + how many were removed and which files are root's again. Nothing goes to stdin. + +### Fixed +- `AptProxyMixin` knows that apt's https sources take the http proxy too: apt's https + method asks `Acquire::https::Proxy` and then `Acquire::http::Proxy`, for the host and + then for all. An `00proxy` with the http line alone sent download.docker.com and + every other https source to apt-cacher-ng, which answers `403 CONNECT denied` without + a `PassThroughPattern`, and the whole upgrade failed (NetOrk/netork#1811). + - The report carries `https_proxy`, `https_per_host` and `https_settings`; + `https_proxy_for(report, host)` is the proxy apt takes for one https host. + - `check_https_source()` fetches one https source's `InRelease` the way apt does; + None where no https source goes through a proxy. + - `configure_package_proxy` gains an `https` step: one https source through the new + proxy. Where the proxy refuses it, `00proxy` also says `Acquire::https::Proxy + "DIRECT";`, later https lines to the same proxy are commented out, and the source + is fetched direct. An HTTP error from the mirror counts as let through. + - `apt_fetch_command` takes https URLs; `direct` sets the host's https setting too. + - `comment_out_command(..., scheme="https")`. + +## [4.22.0] - 2026-10-23 + +### Added +- `AptProxyMixin` (`napalm_device_types.apt_proxy`): apt's proxy on a + Debian-family host, for a driver to mix in (NetOrk/netork#1397). + - `get_package_proxy()` is the proxy apt really fetches through, as + `apt-config dump` says, and no longer the content of one file. A proxy the + Debian or Ubuntu installer set elsewhere wins over `00proxy`, and was missed. + - `get_package_proxy_report()` adds where it is set: every line of apt's + configuration that sets it, in the order apt reads them, the per-host + proxies, and the sources fetched over http and over https. + - `check_package_proxy(url, expect)` fetches a page from the host with apt's + own downloader, past any proxy, and says whether the host reached it and + whether the page says what you expected. + - `configure_package_proxy(url)` writes the proxy, comments out every other + line that sets a different one, reads the result back and fetches one + source through the proxy. It fails when apt still fetches elsewhere. + - `same_proxy_url()` compares two proxy URLs by scheme, host and port. + +## [4.21.1] - 2026-10-22 + +The first release on PyPI since 0.1.0. + +### Changed +- The package follows the NAPALM driver standard + (https://git.netork.io/NAPALM/driver-standard): CI on Gitea runs ruff, strict + mypy, the tests on Python 3.10–3.13 and at the lowest declared versions, + bandit, pip-audit and a build check, and a release tag publishes to PyPI. +- Type annotations use builtin generics (`list[str]`, `X | None`) instead of + `typing.List` and `Optional`. They mean the same to a type checker. +- The project links point at git.netork.io instead of GitHub. diff --git a/README.md b/README.md index 1f90cef..ea8ba97 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ Instead of inheriting directly from `napalm.base.NetworkDriver`, a driver can in ```python # without napalm-device-types -class OpenWrtDriver(NetworkDriver): - ... +class OpenWrtDriver(NetworkDriver): ... + # with napalm-device-types from napalm_device_types import AccessPointDriver -class OpenWrtDriver(AccessPointDriver): - ... + +class OpenWrtDriver(AccessPointDriver): ... ``` ## Why? @@ -29,8 +29,9 @@ OpenMediaVault box is a NAS built on Debian. So a driver inherits **one role bas role its device fills**, and **the order it lists them in is the ranking**: ```python -class QnapQtsDriver(StorageDriver, HypervisorDriver, LinuxDriver): - ... # primary_role_of(...) == "storage" +class QnapQtsDriver( + StorageDriver, HypervisorDriver, LinuxDriver +): ... # primary_role_of(...) == "storage" ``` `roles_of(cls)`, `role_keys_of(cls)` and `primary_role_of(cls)` read that back. Nothing @@ -54,10 +55,12 @@ host OS's load; it has no `HealthMetricsMixin` (SNMP) and no command channel. ```python class StorageDriver(DeviceTypeDriver): """Contract, not code.""" + ROLE: str = "storage" TYPE_LABEL: str = "Storage" - if TYPE_CHECKING: # nothing exists at runtime + if TYPE_CHECKING: # nothing exists at runtime + def get_disks(self) -> List[PhysicalDiskDict]: ... ``` @@ -82,14 +85,57 @@ base; a mixin listed after it loses the MRO to NAPALM's stub. For the same reaso `hasattr` says nothing about NAPALM's getters, and the OS role's account listing is `get_os_users`, not `get_users` -- NAPALM's `get_users` has a different shape. +### NAPALM's getters: an unknown ARP age is `-1.0` + +NAPALM requires an `age` on every entry of `get_arp_table` and +`get_ipv6_neighbors_table`: seconds since the device last confirmed that neighbour. +Many devices do not report one (`ip neigh`, `/proc/net/arp`, a router's host list). +A driver that cannot read the age reports `ARP_AGE_UNKNOWN` (`-1.0`, as NAPALM's IOS +driver does), never `0.0`: `0.0` means "confirmed just now", and an application that +ranks sightings by freshness takes it at its word (NetOrk/netork#1327). An expiry +countdown (FreeBSD's `expires in 1180 seconds`) is not an age either. A driver whose +device does report the age keeps reporting it. + ## Function classes: what a device *can do* Behaviour shared across roles lives in a function class, exactly once, and a role base is a thin bundle over them — `PackageManagementMixin`, `HealthMetricsMixin`, `ServiceControlMixin`, `UpdateMixin`, `NatVpnMixin`, `MacAclMixin`, `FirewallRuleMixin`, `DhcpServerMixin`, `PingSweepMixin`, `ConfigLifecycleMixin`, `InterfaceFilterMixin`. -`HostRebootMixin` (`reboot_host`) is mixed into `DeviceTypeDriver` itself, since any -device may be restartable; like the others it only declares. +`HostRebootMixin` (`reboot_host`) and `HostShutdownMixin` (`shutdown_host`) are mixed +into `DeviceTypeDriver` itself, since any device may be restartable or able to shut itself +down; like the others they only declare. `VmStopRequestMixin` (`begin_stop_vm`) asks for a +VM's clean shutdown without waiting for it — a hypervisor driver mixes it in when it can. + +**Fixes are templates** (`napalm_device_types.fixes`, since 4.13). A change the +application wants made on a device runs as fixed steps the driver implements, with values +the caller supplies: `SnmpAgentMixin.configure_snmp_agent(settings)` (install, configure, +firewall, start), `LldpAgentMixin.configure_lldp_agent(interfaces)` (install, write the +interfaces, enable and restart; the driver writes only the interfaces), +`EnsurePackageMixin.ensure_package(name)`, `PackageProxyMixin.configure_package_proxy(url)` and +`ContainerEngineMixin.grant_container_engine_access(user)`. A driver whose device takes the +change in one call overrides the template method. A driver mixes in the fixes its device +can take; none is on a role base. They replace `run_device_action`, which is deprecated. + +**SNMP is the application's** (`docs/STRUCTURE.md`). `HealthMetricsMixin` is deprecated +since 4.12 and goes in 5.0. A driver does not collect over SNMP; a driver whose device +publishes its health values under OIDs of its own declares them as data: + +```python +class MySwitchDriver(SwitchDriver): + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = { + "uptime": {"oids": ["1.3.6.1.2.1.1.3.0"]}, + "cpu": {"oids": ["", ""]}, + "memory": {"total": {"oids": [""]}, "free": {"oids": [""]}, "unit": "KiB"}, + } +``` + +A value is `oids` (the first that answers wins) or a `table` folded by `first`, `average` +or `max`, with an optional `pattern` for an agent that answers with text. What the profile +does not name, the application reads from the standard UCD-MIB and IF-MIB. +A value the agent gets wrong is `{"unknown": True}`: the application then reports no value +for it, rather than the standard reading or a default (OpenBSD's net-snmp reports its CPU +as never idle). `KernelFactsMixin` (`get_kernel_facts`) is the exception that is mixed in by a driver rather than by a role base: what a Linux kernel has built and loaded is read the same way @@ -97,6 +143,22 @@ everywhere, so the command and its parse are concrete here and a driver supplies `_run_kernel_facts_command`. `OSDriver` does not carry it — a Windows host is an OS driver too, and `hasattr(driver, "get_kernel_facts")` has to stay truthful. +`ServicePackagesMixin` (`get_service_packages`) is mixed in the same way: which installed +packages each systemd service is made of — the one shipping its unit file and the one +shipping the binary its `ExecStart` runs, by `dpkg -S` or `rpm -qf`, not the libraries +that binary links against. An application needs it to let a statement about a service +(the postgresql unit) reach the findings on its packages. A driver supplies +`_run_service_packages_command`. + +`AptProxyMixin` (`napalm_device_types.apt_proxy`, since 4.22) is `PackageProxyMixin` for a +Debian-family host, mixed in the same way. What apt fetches through is what `apt-config dump` +says, not one file: apt reads `apt.conf.d` in order and `apt.conf` last, and the last value +wins (`get_package_proxy`, `get_package_proxy_report` with every line that sets it). +`check_package_proxy(url, expect)` fetches a page with apt's own downloader, past any proxy. +`configure_package_proxy(url)` writes `00proxy`, comments out every other line that sets a +different proxy, reads the result back and fetches one http source's `InRelease` through it. +A driver supplies `_run_apt_command`, `_run_apt_root_command` and `_write_apt_file`. + `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer kernel of the running flavour installed), whether it patches itself (unattended-upgrades, @@ -117,6 +179,17 @@ without `ss` is read with `netstat -lntup` (OpenWrt's busybox, old net-tools); o cgroup names the procd service (`/services//`). A host with neither raises `ListeningSocketsUnavailable`. +`HostFirewallMixin` (`get_host_firewall`) reads a Linux host's own netfilter rules the same +way: `nft -j list ruleset`, with the tables iptables-nft manages taken from `iptables-save` +instead, because nft shows their extensions (`-m conntrack`, `-m set`) only as an opaque `xt`. +A legacy iptables is a ruleset of its own and is kept beside nft's. Every chain says its family, +table, hook, priority and policy; every rule is a list of matches (`iifname`, `saddr`, `dport`, +`ct_state`, ...) and a verdict. A condition it does not model is an `unknown` match with its +text, never left out. A driver supplies `_run_host_firewall_command(command, privileged=)`; both +tools need root, so a host where neither could be read raises `HostFirewallUnavailable` rather +than reading as one without rules. Whether the rules keep a service unreachable is the +consumer's call. + `VirtualIpsMixin` (`get_virtual_ips`) reads the addresses a host's HA configuration lets float between machines: vip-manager (5.x's `--config` YAML, 1.x's `-ip=${VIP_IP}` resolved through the unit's environment) and keepalived (`virtual_ipaddress` of every @@ -128,6 +201,40 @@ A mechanism it could not read completely is `None` ("did not look"), a host with `[]`. The command is about 4 kB, so a driver sends it on an exec channel (`_run_virtual_ips_command(command, privileged=)`), not typed into a shell. +`ServiceClustersMixin` (`get_service_clusters`) reads the configuration that proves two +hosts run one service together -- not merely that they answer at one address: Patroni +(`scope`, `name` and `restapi.connect_address` from its YAML, then the members and roles +its REST API's `/cluster` lists from the DCS), adguardhome-sync (`ORIGIN_URL` and +`REPLICA*_URL` of its container: the origin pushes to each replica) and BunkerWeb (a +worker's `API_WHITELIST_IP` names the manager it takes its configuration from, a manager's +`BUNKERWEB_INSTANCES` its workers). Each member carries its role as the mechanism words it +and the index of the member it takes its configuration from (`config_from`). The same +configuration holds the superuser password, the sync's logins and BunkerWeb's database URI, +so awk programs filter **on the host**: allowlisted fields only, a URL without its +`user:password@`, and the DCS endpoints stay on the host. Root first (Patroni's YAML is the +postgres user's), then without it. `None` per mechanism is "did not look", `[]` "has none". +The container engine is asked through `_container_engine_binary` where the driver has one. +A driver sends the command on an exec channel (`_run_service_clusters_command(command, +privileged=)`). + +`get_dhcp_ha` (declared on `DhcpServerMixin`, optional) is what a DHCP server's own HA +configuration says: whether HA is on, which peer this server is, each peer's name, role -- +as the server words it, not a shared vocabulary -- and, where the server has one, the URL +its HA listener answers on. It is configuration, not state: whether the pair is in hot +standby right now is not in it. `ConfigSyncMixin` (`get_config_sync`) is where a host +pushes its configuration (`peer`, None for no one) and for which services, by the names +`get_services` reports; sync items the driver has no service name for stay visible under +`unmapped`. A driver mixes it in -- no role promises it. Both getters raise when they could +not read; neither answers "off" for "could not ask". What the facts mean for a pair (hot +standby, which side to write to) is the caller's to work out. + +A firewall's CARP addresses are a third mechanism, `carp`: a firewall driver reads them from +its own API and reports them under the reading's `carp` key (`vip_manager` and `keepalived` +then None). What a CARP state means is protocol knowledge and stays in that driver. The +contract only carries the result: unlike the other two, a CARP entry says who holds the +address (`holds`), because a firewall's interface list need not show it. `holds` None +leaves the holder to the interfaces. + `SystemIdentityMixin` (`get_system_identity`) reads a host's SMBIOS identity -- system UUID, product serial and board serial from `/sys/class/dmi/id` -- which is what ties a server to its BMC (`ManagementControllerDriver.get_hardware_inventory` reports the same values from @@ -149,6 +256,16 @@ A driver mixes it in and supplies `run_command`; a platform that regenerates ssh itself (an appliance with its own database) must not, or opts out with `harden_ssh_server = None`. +`SudoHistoryMixin` (`get_history_sudo_lines`, `remove_history_sudo_lines`) deals with what +a driver left behind when it typed `echo '' | sudo -S -p "" …` into an interactive +shell: that line, password and all, in the login user's shell history, and files its partial +sudo wrote as the login user. The driver names the marker its lines carry +(`SUDO_HISTORY_MARKER`) and those files (`SUDO_ROOT_FILES`). The getter counts the lines per +history file with `grep -c`, as the login user; the fix deletes them with `sed -i` and +overwrites the backup with `shred -u`, then gives the files back to root (regular files the +login user owns, never a symbolic link). The password is never read, sent or written. A +driver mixes it in and supplies `run_command`. + **Update readers raise when they cannot read.** `get_available_updates` returns an empty list only when nothing is pending; netOrk keeps "pending since" per package, and an empty list for "don't know" would reset it. @@ -194,6 +311,47 @@ NAPALM's own getters (`get_facts`, `get_interfaces`, `ping`, `get_config`) are n wrapped in a template — they belong to NAPALM, and code outside this repo relies on their contract. +### SSH host keys: trust on first use + +A driver that accepts any host key logs in with the device's credentials at whatever +answers on the device's address (NetOrk/netork#1167). The store of trusted keys is the +caller's; `ssh_host_keys.HostKeyCheck` is how every SSH driver verifies against it: + +- **In:** `optional_args["ssh_host_keys"]` = `{"22": {"key_type": "ssh-ed25519", "key": ""}}`, + by port. The driver takes the key for the port it connects to. With one, any other key is + refused *before* authentication and the connection raises `SshHostKeyMismatch` (a + `ConnectionException`). Without one, the first key is accepted. +- **Out:** `optional_args["ssh_host_key_observer"]`, a callable, gets one dict per + connection: `{"host", "port", "seen": {"key_type", "key", "fingerprint"}, "expected", "accepted"}`. + `accepted` is False only for a refused key. A failing observer is logged, never raised. +- **paramiko:** `HostKeyCheck(optional_args, host, port).connect(client, **connect_kwargs)`. +- **netmiko:** `with check.netmiko(self.netmiko_optional_args) as kwargs: ConnectHandler(..., **kwargs)`, + then `check.observe(conn.remote_conn_pre.get_transport())`. The known key reaches netmiko as a + `known_hosts` file that exists only inside the block, with `ssh_strict=True`. + +A caller that passes neither key gets first use on every connection, as before; a driver +that ignores both still connects. Old and new versions on either side keep working. + +### TLS certificates: trust on first use + +The same for a driver's HTTPS API. Devices ship self-signed certificates, so a CA check +fails on all of them and turning verification off trusts whatever answers; +`tls_certificates.CertificateCheck` pins the certificate by its fingerprint instead +(NAPALM/napalm-hpe-aruba-procurve#17). No CA is consulted. + +- **In:** `optional_args["tls_fingerprints"]` = `{"443": "SHA256:AB:CD:…"}`, by port: the + SHA-256 of the DER certificate as `openssl x509 -fingerprint -sha256` prints it (prefix, + colons and case optional). With one, any other certificate is refused *before* a request + carries credentials and the connection raises `TlsCertificateMismatch` (a + `ConnectionException` naming both fingerprints). Without one, the first certificate is + accepted. A driver does not fall back to another transport on a mismatch. +- **Out:** `optional_args["tls_certificate_observer"]`, a callable, gets one dict per + connection: `{"host", "port", "seen": {"fingerprint"}, "expected", "accepted"}`. +- **requests:** `check = CertificateCheck(optional_args, host, port)`; + `check.verify_server(timeout)` before the login, `check.pin(session)` so every later + connection of the session must present that certificate, the login inside + `with check.refusing():`, then `check.observe()`. + ## Design principle: generic vs. device-specific logic When adding behavior to a device-type base class, split it along one line: **would @@ -213,20 +371,27 @@ generic data shapes belong here. Only the actual device communication belongs in **Worked example — firewall rule diff/apply** (`FirewallRuleMixin`, in `FirewallDriver`): ```python -class FirewallRuleMixin: # mixed into FirewallDriver +class FirewallRuleMixin: # mixed into FirewallDriver # Declared only — every driver implements its own device communication. if TYPE_CHECKING: + def get_firewall_rules(self) -> List[FirewallRuleDict]: ... - def apply_firewall_rule(self, rule: FirewallRuleDict, *, uuid: Optional[str] = None) -> Dict[str, Any]: ... + def apply_firewall_rule( + self, rule: FirewallRuleDict, *, uuid: Optional[str] = None + ) -> Dict[str, Any]: ... def commit_firewall_rules(self) -> Dict[str, Any]: ... # Concrete — the matching/comparison/orchestration algorithm is identical # for every firewall vendor, so it lives here once. - def diff_firewall_rules(self, desired: List[FirewallRuleDict]) -> FirewallRuleDiffDict: - ... # matches self.get_firewall_rules() against `desired` by description + def diff_firewall_rules( + self, desired: List[FirewallRuleDict] + ) -> ( + FirewallRuleDiffDict + ): ... # matches self.get_firewall_rules() against `desired` by description - def apply_firewall_ruleset(self, desired: List[FirewallRuleDict]): - ... # computes the diff, calls apply_firewall_rule() per change, commits + def apply_firewall_ruleset( + self, desired: List[FirewallRuleDict] + ): ... # computes the diff, calls apply_firewall_rule() per change, commits ``` The same split applies to `DhcpServerMixin`: `get_dhcp_reservations`/ @@ -270,7 +435,7 @@ detection vs. presentation there. pip install napalm-device-types ``` -Requires Python ≥ 3.10 (tested up to 3.13) and NAPALM ≥ 4.0. +Requires Python ≥ 3.12 (tested up to 3.13) and NAPALM ≥ 4.0. ## Available base classes @@ -322,8 +487,8 @@ HTTP_FINGERPRINT = [ ```python from napalm_device_types import AccessPointDriver -class OpenWrtDriver(AccessPointDriver): +class OpenWrtDriver(AccessPointDriver): def get_wireless_clients(self): # return List[WirelessClientDict] ... @@ -338,8 +503,8 @@ class OpenWrtDriver(AccessPointDriver): ```python from napalm_device_types import SwitchDriver -class CiscoIOSDriver(SwitchDriver): +class CiscoIOSDriver(SwitchDriver): def get_spanning_tree(self): # return Dict[str, SpanningTreeDict] ... @@ -354,8 +519,8 @@ class CiscoIOSDriver(SwitchDriver): ```python from napalm_device_types import FirewallDriver -class PfSenseDriver(FirewallDriver): +class PfSenseDriver(FirewallDriver): def get_nat_translations(self): # return List[NATTranslationDict] ... @@ -375,13 +540,20 @@ class PfSenseDriver(FirewallDriver): ```python from napalm_device_types import HypervisorDriver -class ProxmoxDriver(HypervisorDriver): +class ProxmoxDriver(HypervisorDriver): def get_vms(self): # return List[VMDict] ... - def create_vm_snapshot(self, name, snapshot, description="", include_memory=False): + def create_vm_snapshot(self, name, snapshot, description="", include_memory=False): ... + + def get_network_targets(self): + # return List[NetworkTargetDict]: bridges and vnets a new VM's NIC may + # attach to; a bridge may name its physical NICs in `uplinks`, one group + # per way out: [["eno1", "eno2"]] for a bond, [["eno1"]] for a single NIC, + # no internal or VLAN ports -- leave the key out when they cannot be + # determined ... def get_vm_cpu_types(self): @@ -399,15 +571,18 @@ any other. A QEMU-based driver takes `provisioning.QEMU_GUEST_AGENTS` (Linux, FreeBSD, OpenBSD). `provisioning.network_config()` writes the network-config v2 every cloud-init flavour reads (FreeBSD's nuageinit reads no other), and `provisioning.split_compression()` says how a packed image is -unpacked; both are the same for every hypervisor. +unpacked; both are the same for every hypervisor. A creation that fails once +the VM exists raises `VMProvisionIncomplete`, a `RuntimeError` that names the +half-built VM (`vmid`, `node`) for the caller to `destroy_vm`, with the failure +as its cause. ### OS / Linux ```python from napalm_device_types import OSDriver -class LinuxDriver(OSDriver): +class LinuxDriver(OSDriver): def get_packages(self): # return List[PackageDict] ... @@ -434,8 +609,8 @@ class LinuxDriver(OSDriver): ```python from napalm_device_types import StorageDriver -class TrueNASDriver(StorageDriver): +class TrueNASDriver(StorageDriver): def get_disks(self): # return List[PhysicalDiskDict] ... @@ -461,7 +636,7 @@ from napalm_device_types.models import ( ## Development ```bash -git clone https://github.com/chrismanivong/napalm-device-types.git +git clone https://git.netork.io/NAPALM/napalm-device-types.git cd napalm-device-types pip install -e ".[dev]" ``` @@ -474,10 +649,15 @@ mypy napalm_device_types ## Contributing -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/router-driver`) -3. Implement your changes -4. Open a Pull Request +This package follows the +[NAPALM driver standard](https://git.netork.io/NAPALM/driver-standard/src/branch/main/STANDARDS.md): +test first, ruff and strict mypy, Conventional Commits, one pull request per change +against `main`. CI checks what a tool can check: + +```bash +pip install "napalm-standard @ git+https://git.netork.io/NAPALM/driver-standard.git" +napalm-standard check && napalm-standard ratchet +``` ## License diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 5eb32f9..68f28d1 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -37,31 +37,48 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.channel.CommandChannelMixin` * :class:`~napalm_device_types.config_lifecycle.ConfigLifecycleMixin` +* :class:`~napalm_device_types.config_sync.ConfigSyncMixin` * :class:`~napalm_device_types.container_engine.ContainerEngineMixin` * :class:`~napalm_device_types.dhcp.DhcpServerMixin` * :class:`~napalm_device_types.firewall_rules.FirewallRuleMixin` * :class:`~napalm_device_types.health_metrics.HealthMetricsMixin` * :class:`~napalm_device_types.host_status.HostStatusMixin` * :class:`~napalm_device_types.host_reboot.HostRebootMixin` +* :class:`~napalm_device_types.host_shutdown.HostShutdownMixin` * :class:`~napalm_device_types.interface_filter.InterfaceFilterMixin` * :class:`~napalm_device_types.kernel.KernelFactsMixin` * :class:`~napalm_device_types.listening.ListeningSocketsMixin` +* :class:`~napalm_device_types.local_accounts.LocalAccountsMixin` +* :class:`~napalm_device_types.host_firewall.HostFirewallMixin` * :class:`~napalm_device_types.mac_acl.MacAclMixin` * :class:`~napalm_device_types.nat_vpn.NatVpnMixin` * :class:`~napalm_device_types.packages.PackageManagementMixin` * :class:`~napalm_device_types.ping_sweep.PingSweepMixin` +* :class:`~napalm_device_types.service_clusters.ServiceClustersMixin` +* :class:`~napalm_device_types.service_packages.ServicePackagesMixin` +* :class:`~napalm_device_types.sudo_history.SudoHistoryMixin` * :class:`~napalm_device_types.services.ServiceControlMixin` * :class:`~napalm_device_types.systemd.SystemdServicesMixin` * :class:`~napalm_device_types.updates.UpdateMixin` * :class:`~napalm_device_types.virtual_ips.VirtualIpsMixin` +* :class:`~napalm_device_types.vm_stop_request.VmStopRequestMixin` Introspection -- :func:`~napalm_device_types.roles.roles_of`, :func:`~napalm_device_types.roles.role_keys_of` and :func:`~napalm_device_types.roles.primary_role_of`. """ -from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec, rules_max_weight from napalm_device_types.access_point import AccessPointDriver +from napalm_device_types.apt_proxy import ( + APT_PROXY_COMMAND, + AptProxyMixin, + https_proxy_for, + parse_apt_fetch, + parse_apt_proxy, + same_proxy_url, +) +from napalm_device_types.arp import ARP_AGE_UNKNOWN +from napalm_device_types.base import DeviceTypeDriver, FingerprintRule, PortSpec, rules_max_weight from napalm_device_types.channel import ( ByteStream, CommandChannelMixin, @@ -70,73 +87,130 @@ from napalm_device_types.channel import ( open_stream_on_transport, run_on_transport, ) +from napalm_device_types.clusters import ( + BackupDict, + BackupJobDict, + ClusterDict, + ClusterNetworkDict, + ClusterNodeDict, + ClusterStorageDict, + HAResourceDict, +) from napalm_device_types.config_lifecycle import ConfigLifecycleMixin +from napalm_device_types.config_sync import ConfigSyncMixin from napalm_device_types.container_engine import ( ContainerEngineConnection, ContainerEngineMixin, ContainerEngineUnavailable, ) from napalm_device_types.dhcp import DhcpServerMixin, normalize_cidr, normalize_mac +from napalm_device_types.dns_forward import DnsForwardMixin from napalm_device_types.firewall import FirewallDriver -from napalm_device_types.hypervisor import HypervisorDriver -from napalm_device_types.os import OSDriver from napalm_device_types.firewall_rules import FirewallRuleMixin +from napalm_device_types.firmware import FirmwareOfferMixin +from napalm_device_types.fixes import ( + EnsurePackageMixin, + LldpAgentMixin, + PackageProxyMixin, + SnmpAgentMixin, + run_fix_steps, +) from napalm_device_types.health_metrics import HealthMetricsMixin +from napalm_device_types.host_firewall import ( + HostFirewallMixin, + HostFirewallUnavailable, + parse_host_firewall, +) from napalm_device_types.host_reboot import HostRebootMixin +from napalm_device_types.host_shutdown import HostShutdownMixin +from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status +from napalm_device_types.hypervisor import HypervisorDriver from napalm_device_types.interface_filter import InterfaceFilterMixin from napalm_device_types.kernel import KERNEL_FACTS_COMMAND, KernelFactsMixin, parse_kernel_facts from napalm_device_types.lag import add_lag_interfaces +from napalm_device_types.licenses import LicenseMixin from napalm_device_types.listening import ( LISTENING_SOCKETS_COMMAND, ListeningSocketsMixin, ListeningSocketsUnavailable, parse_listening_sockets, ) +from napalm_device_types.local_accounts import ( + LocalAccountChangeDict, + LocalAccountDict, + LocalAccountsMixin, +) from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver from napalm_device_types.models import ( POWER_ACTIONS, + ConfigSyncDict, + DhcpHaDict, + DhcpHaPeerDict, + DnsForwardDict, EventLogEntryDict, FirmwareDict, + FirmwareOfferDict, + FixResultDict, + FixStepDict, + FixStepOutcomeDict, HardwareInventoryDict, + HypervisorHostDict, + LicenseDict, ManagementProtocolsDict, + OspfDict, + OspfInterfaceDict, PowerAction, PowerStateDict, + SnmpAgentSettingsDict, + UpdateProgressDict, ) from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.packages import PackageManagementMixin -from napalm_device_types.phone import PhoneDriver -from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping -from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of -from napalm_device_types.host_status import HOST_STATUS_COMMAND, HostStatusMixin, parse_host_status +from napalm_device_types.os import OSDriver +from napalm_device_types.ospf import OspfMixin from napalm_device_types.package_updates import ( APT_UPGRADABLE_COMMAND, DNF_SECURITY_COMMAND, parse_apt_upgradable, parse_dnf_security, ) -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.terminal import strip_terminal_codes -from napalm_device_types.systemd import ( - SYSTEMD_SERVICES_COMMAND, - SystemdServicesMixin, - SystemdUnavailable, - parse_systemd_services, -) -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.system_identity import ( - SYSTEM_IDENTITY_COMMAND, - SystemIdentityMixin, - is_placeholder_identity, - parse_system_identity, -) -from napalm_device_types.virtual_ips import ( - VIRTUAL_IPS_COMMAND, - VirtualIpsMixin, - parse_virtual_ips, -) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.phone import PhoneDriver +from napalm_device_types.ping_sweep import PingSweepMixin, driver_supports_ping +from napalm_device_types.provisioning import VMProvisionIncomplete from napalm_device_types.residential_gateway import ResidentialGatewayDriver +from napalm_device_types.roles import primary_role_of, role_keys_of, roles_of +from napalm_device_types.service_clusters import ( + SERVICE_CLUSTERS_COMMAND, + ServiceClustersMixin, + parse_service_clusters, + service_clusters_command, +) +from napalm_device_types.service_packages import ( + SERVICE_PACKAGES_COMMAND, + ServicePackagesMixin, + parse_service_packages, +) +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.snmp_health_profile import ( + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, +) +from napalm_device_types.ssh_host_keys import ( + KNOWN_KEYS_ARG as SSH_HOST_KEYS_ARG, +) +from napalm_device_types.ssh_host_keys import ( + OBSERVER_ARG as SSH_HOST_KEY_OBSERVER_ARG, +) +from napalm_device_types.ssh_host_keys import ( + HostKeyCheck, + HostKeyObservation, + SshHostKey, + SshHostKeyMismatch, +) from napalm_device_types.ssh_server import ( SSH_ALGORITHM_CLASSES, OpenSshServerMixin, @@ -145,9 +219,74 @@ from napalm_device_types.ssh_server import ( plan_ssh_hardening, ) from napalm_device_types.storage import StorageDriver +from napalm_device_types.sudo_history import ( + HistoryCleanupDict, + HistorySudoLinesDict, + SudoHistoryCleanupDict, + SudoHistoryMixin, +) from napalm_device_types.switch import SwitchDriver +from napalm_device_types.system_identity import ( + SYSTEM_IDENTITY_COMMAND, + SystemIdentityMixin, + is_placeholder_identity, + parse_system_identity, +) +from napalm_device_types.systemd import ( + SYSTEMD_SERVICES_COMMAND, + SystemdServicesMixin, + SystemdUnavailable, + parse_systemd_services, +) +from napalm_device_types.terminal import strip_terminal_codes +from napalm_device_types.updates import UpdateMixin +from napalm_device_types.virtual_ips import ( + VIRTUAL_IPS_COMMAND, + VirtualIpsMixin, + parse_virtual_ips, +) +from napalm_device_types.vm_migration import MigrationPreflightDict +from napalm_device_types.vm_stop_request import VmStopRequestMixin __all__ = [ + "BackupDict", + "BackupJobDict", + "ClusterDict", + "ClusterNetworkDict", + "ClusterNodeDict", + "ClusterStorageDict", + "HAResourceDict", + "MigrationPreflightDict", + "HypervisorHostDict", + "DnsForwardDict", + "DnsForwardMixin", + "FirmwareOfferDict", + "OspfDict", + "OspfInterfaceDict", + "OspfMixin", + "UpdateProgressDict", + "FirmwareOfferMixin", + "LicenseDict", + "LicenseMixin", + "EnsurePackageMixin", + "FixResultDict", + "FixStepDict", + "FixStepOutcomeDict", + "LldpAgentMixin", + "PackageProxyMixin", + "APT_PROXY_COMMAND", + "AptProxyMixin", + "https_proxy_for", + "parse_apt_fetch", + "parse_apt_proxy", + "same_proxy_url", + "SnmpAgentMixin", + "SnmpAgentSettingsDict", + "run_fix_steps", + "SnmpHealthProfile", + "SnmpInterfacesSpec", + "SnmpMemorySpec", + "SnmpValueSpec", "AccessPointDriver", "ByteStream", "CommandChannelMixin", @@ -159,6 +298,10 @@ __all__ = [ "open_stream_on_transport", "run_on_transport", "ConfigLifecycleMixin", + "ConfigSyncDict", + "ConfigSyncMixin", + "DhcpHaDict", + "DhcpHaPeerDict", "DeviceTypeDriver", "DhcpServerMixin", "FingerprintRule", @@ -169,10 +312,12 @@ __all__ = [ "HOST_STATUS_COMMAND", "HostStatusMixin", "HostRebootMixin", + "HostShutdownMixin", "HypervisorDriver", "InterfaceFilterMixin", "MacAclMixin", "ManagementControllerDriver", + "ARP_AGE_UNKNOWN", "POWER_ACTIONS", "PowerAction", "PowerStateDict", @@ -193,10 +338,19 @@ __all__ = [ "KernelFactsMixin", "KERNEL_FACTS_COMMAND", "parse_kernel_facts", + "SERVICE_PACKAGES_COMMAND", + "ServicePackagesMixin", + "parse_service_packages", "LISTENING_SOCKETS_COMMAND", + "HostFirewallMixin", + "HostFirewallUnavailable", + "parse_host_firewall", "ListeningSocketsMixin", "ListeningSocketsUnavailable", "parse_listening_sockets", + "LocalAccountsMixin", + "LocalAccountDict", + "LocalAccountChangeDict", "PhoneDriver", "PingSweepMixin", "PortSpec", @@ -211,6 +365,12 @@ __all__ = [ "UpdateMixin", "VIRTUAL_IPS_COMMAND", "VirtualIpsMixin", + "VmStopRequestMixin", + "VMProvisionIncomplete", + "SERVICE_CLUSTERS_COMMAND", + "ServiceClustersMixin", + "parse_service_clusters", + "service_clusters_command", "SystemIdentityMixin", "SYSTEM_IDENTITY_COMMAND", "is_placeholder_identity", @@ -220,6 +380,16 @@ __all__ = [ "SshHardeningPlan", "SshHardeningResult", "plan_ssh_hardening", + "SudoHistoryMixin", + "HistorySudoLinesDict", + "HistoryCleanupDict", + "SudoHistoryCleanupDict", + "SSH_HOST_KEYS_ARG", + "SSH_HOST_KEY_OBSERVER_ARG", + "HostKeyCheck", + "HostKeyObservation", + "SshHostKey", + "SshHostKeyMismatch", "parse_virtual_ips", "add_lag_interfaces", "driver_supports_ping", diff --git a/napalm_device_types/_matching.py b/napalm_device_types/_matching.py index 8628053..5cab64d 100644 --- a/napalm_device_types/_matching.py +++ b/napalm_device_types/_matching.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Keying live and desired entries for a diff, refusing what cannot be matched. A diff matches a desired entry to a live one by a key (a rule's description, a @@ -8,18 +7,16 @@ name a single entry, and guessing would overwrite one the caller never meant. from __future__ import annotations -from typing import Callable, Dict, Iterable, List, Set, Tuple, TypeVar - -T = TypeVar("T") +from collections.abc import Callable, Iterable -def index_live(entries: Iterable[T], key: Callable[[T], str]) -> Tuple[Dict[str, T], Set[str]]: +def index_live[T](entries: Iterable[T], key: Callable[[T], str]) -> tuple[dict[str, T], set[str]]: """Live entries by key, and the keys more than one of them holds. Entries without a key are left out: they are not the caller's to match. """ - by_key: Dict[str, T] = {} - shared: Set[str] = set() + by_key: dict[str, T] = {} + shared: set[str] = set() for entry in entries: k = key(entry) if not k: @@ -32,10 +29,12 @@ def index_live(entries: Iterable[T], key: Callable[[T], str]) -> Tuple[Dict[str, return by_key, shared -def check_desired(entries: List[T], key: Callable[[T], str], shared: Set[str], what: str) -> None: +def check_desired[T]( + entries: list[T], key: Callable[[T], str], shared: set[str], what: str +) -> None: """``ValueError`` for a desired entry with no key, a key given twice, or a key that names more than one live entry.""" - seen: Set[str] = set() + seen: set[str] = set() for entry in entries: k = key(entry) if not k: diff --git a/napalm_device_types/_ucd_metrics.py b/napalm_device_types/_ucd_metrics.py index 7148f9d..9dc44f0 100644 --- a/napalm_device_types/_ucd_metrics.py +++ b/napalm_device_types/_ucd_metrics.py @@ -1,41 +1,43 @@ """Shared SNMP metric collection helpers for UCD-MIB and IF-MIB.""" + from __future__ import annotations import asyncio import re -from typing import Any, Awaitable, Callable, Dict, Optional +from collections.abc import Awaitable, Callable +from typing import Any # ── UCD-MIB OIDs ────────────────────────────────────────────────────────────── -OID_SYS_UPTIME = "1.3.6.1.2.1.25.1.1.0" # hrSystemUptime (centiseconds) -OID_CPU_IDLE = "1.3.6.1.4.1.2021.11.11.0" # UCD ssCpuIdle (%) -OID_MEM_TOTAL = "1.3.6.1.4.1.2021.4.5.0" # memTotalReal (kB) -OID_MEM_FREE = "1.3.6.1.4.1.2021.4.6.0" # memAvailReal (kB) -OID_MEM_BUFFER = "1.3.6.1.4.1.2021.4.14.0" # memBuffer (kB) -OID_MEM_CACHED = "1.3.6.1.4.1.2021.4.15.0" # memCached (kB) -OID_SWAP_TOTAL = "1.3.6.1.4.1.2021.4.3.0" # memTotalSwap (kB) -OID_SWAP_AVAIL = "1.3.6.1.4.1.2021.4.4.0" # memAvailSwap (kB) -OID_LOAD_1 = "1.3.6.1.4.1.2021.10.1.3.1" # laLoad 1-min -OID_LOAD_5 = "1.3.6.1.4.1.2021.10.1.3.2" # laLoad 5-min -OID_LOAD_15 = "1.3.6.1.4.1.2021.10.1.3.3" # laLoad 15-min +OID_SYS_UPTIME = "1.3.6.1.2.1.25.1.1.0" # hrSystemUptime (centiseconds) +OID_CPU_IDLE = "1.3.6.1.4.1.2021.11.11.0" # UCD ssCpuIdle (%) +OID_MEM_TOTAL = "1.3.6.1.4.1.2021.4.5.0" # memTotalReal (kB) +OID_MEM_FREE = "1.3.6.1.4.1.2021.4.6.0" # memAvailReal (kB) +OID_MEM_BUFFER = "1.3.6.1.4.1.2021.4.14.0" # memBuffer (kB) +OID_MEM_CACHED = "1.3.6.1.4.1.2021.4.15.0" # memCached (kB) +OID_SWAP_TOTAL = "1.3.6.1.4.1.2021.4.3.0" # memTotalSwap (kB) +OID_SWAP_AVAIL = "1.3.6.1.4.1.2021.4.4.0" # memAvailSwap (kB) +OID_LOAD_1 = "1.3.6.1.4.1.2021.10.1.3.1" # laLoad 1-min +OID_LOAD_5 = "1.3.6.1.4.1.2021.10.1.3.2" # laLoad 5-min +OID_LOAD_15 = "1.3.6.1.4.1.2021.10.1.3.3" # laLoad 15-min # ── IF-MIB OIDs ─────────────────────────────────────────────────────────────── -OID_IF_DESCR = "1.3.6.1.2.1.2.2.1.2" -OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5" -OID_IF_IN_OCT = "1.3.6.1.2.1.2.2.1.10" +OID_IF_DESCR = "1.3.6.1.2.1.2.2.1.2" +OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5" +OID_IF_IN_OCT = "1.3.6.1.2.1.2.2.1.10" OID_IF_OUT_OCT = "1.3.6.1.2.1.2.2.1.16" -OID_IF_IN_ERR = "1.3.6.1.2.1.2.2.1.14" +OID_IF_IN_ERR = "1.3.6.1.2.1.2.2.1.14" OID_IF_OUT_ERR = "1.3.6.1.2.1.2.2.1.20" # ifXTable: 64-bit octets and Mbps. The 32-bit counters wrap every ~34 s at # 1 Gbit/s and ifSpeed stops at 4294 Mbps; an agent without ifXTable (SNMPv1) # falls back to them. -OID_IF_HC_IN_OCT = "1.3.6.1.2.1.31.1.1.1.6" +OID_IF_HC_IN_OCT = "1.3.6.1.2.1.31.1.1.1.6" OID_IF_HC_OUT_OCT = "1.3.6.1.2.1.31.1.1.1.10" OID_IF_HIGH_SPEED = "1.3.6.1.2.1.31.1.1.1.15" -IF_SKIP_DEFAULT = re.compile(r'^(lo|sit\d|tun\d|docker|veth|br-|virbr)', re.IGNORECASE) +IF_SKIP_DEFAULT = re.compile(r"^(lo|sit\d|tun\d|docker|veth|br-|virbr)", re.IGNORECASE) -SnmpGetFn = Callable[[str], Awaitable[Optional[str]]] -SnmpWalkFn = Callable[[str], Awaitable[Dict[str, str]]] +SnmpGetFn = Callable[[str], Awaitable[str | None]] +SnmpWalkFn = Callable[[str], Awaitable[dict[str, str]]] def ticks_to_seconds(raw: str | None) -> int | None: @@ -48,35 +50,35 @@ def ticks_to_seconds(raw: str | None) -> int | None: def build_if_metrics( - metrics: Dict[str, Any], - descr: Dict[str, str], - speed: Dict[str, str], - in_oct: Dict[str, str], - out_oct: Dict[str, str], - in_err: Dict[str, str], - out_err: Dict[str, str], + metrics: dict[str, Any], + descr: dict[str, str], + speed: dict[str, str], + in_oct: dict[str, str], + out_oct: dict[str, str], + in_err: dict[str, str], + out_err: dict[str, str], *, if_skip: re.Pattern[str] | None = None, tx_err_is_drop: bool = False, - hc_in_oct: Optional[Dict[str, str]] = None, - hc_out_oct: Optional[Dict[str, str]] = None, - high_speed: Optional[Dict[str, str]] = None, + hc_in_oct: dict[str, str] | None = None, + hc_out_oct: dict[str, str] | None = None, + high_speed: dict[str, str] | None = None, ) -> None: if if_skip is None: if_skip = IF_SKIP_DEFAULT - interfaces: Dict[str, Any] = {} + interfaces: dict[str, Any] = {} for idx, name in descr.items(): if if_skip.match(name): continue - iface: Dict[str, Any] = {"name": name} + iface: dict[str, Any] = {"name": name} try: - iface["rx_bytes"] = int((hc_in_oct or {}).get(idx) or in_oct.get(idx) or 0) - iface["tx_bytes"] = int((hc_out_oct or {}).get(idx) or out_oct.get(idx) or 0) - iface["rx_errors"] = int(in_err.get(idx, 0) or 0) + iface["rx_bytes"] = int((hc_in_oct or {}).get(idx) or in_oct.get(idx) or 0) + iface["tx_bytes"] = int((hc_out_oct or {}).get(idx) or out_oct.get(idx) or 0) + iface["rx_errors"] = int(in_err.get(idx, 0) or 0) tx_val = int(out_err.get(idx, 0) or 0) if tx_err_is_drop: iface["tx_errors"] = 0 - iface["tx_drops"] = tx_val + iface["tx_drops"] = tx_val else: iface["tx_errors"] = tx_val high = (high_speed or {}).get(idx) @@ -98,15 +100,33 @@ async def collect_ucd_metrics( *, tx_err_is_drop: bool = False, if_skip: re.Pattern[str] | None = None, -) -> Dict[str, Any]: +) -> dict[str, Any]: """Collect UCD-MIB system metrics + IF-MIB interface counters in parallel.""" ( - uptime_raw, cpu_idle_raw, - mem_total_raw, mem_free_raw, mem_buf_raw, mem_cache_raw, - swap_total_raw, swap_avail_raw, - load1_raw, load5_raw, load15_raw, - ), ( - descr, speed, in_oct, out_oct, in_err, out_err, hc_in_oct, hc_out_oct, high_speed, + ( + uptime_raw, + cpu_idle_raw, + mem_total_raw, + mem_free_raw, + mem_buf_raw, + mem_cache_raw, + swap_total_raw, + swap_avail_raw, + load1_raw, + load5_raw, + load15_raw, + ), + ( + descr, + speed, + in_oct, + out_oct, + in_err, + out_err, + hc_in_oct, + hc_out_oct, + high_speed, + ), ) = await asyncio.gather( asyncio.gather( snmp_get(OID_SYS_UPTIME), @@ -134,7 +154,7 @@ async def collect_ucd_metrics( ), ) - metrics: Dict[str, Any] = {} + metrics: dict[str, Any] = {} secs = ticks_to_seconds(uptime_raw) if secs is not None: @@ -149,14 +169,14 @@ async def collect_ucd_metrics( if mem_total_raw and mem_free_raw: try: total_kb = int(mem_total_raw) - free_kb = int(mem_free_raw) - buf_kb = int(mem_buf_raw) if mem_buf_raw else 0 + free_kb = int(mem_free_raw) + buf_kb = int(mem_buf_raw) if mem_buf_raw else 0 cache_kb = int(mem_cache_raw) if mem_cache_raw else 0 - used_kb = max(0, total_kb - free_kb - buf_kb - cache_kb) + used_kb = max(0, total_kb - free_kb - buf_kb - cache_kb) metrics["memory_total_bytes"] = total_kb * 1024 - metrics["memory_used_bytes"] = used_kb * 1024 - metrics["memory_percent"] = round(used_kb / total_kb * 100, 1) if total_kb else 0.0 - metrics["memory_free_bytes"] = free_kb * 1024 + metrics["memory_used_bytes"] = used_kb * 1024 + metrics["memory_percent"] = round(used_kb / total_kb * 100, 1) if total_kb else 0.0 + metrics["memory_free_bytes"] = free_kb * 1024 except ValueError: pass @@ -164,10 +184,10 @@ async def collect_ucd_metrics( try: stotal = int(swap_total_raw) savail = int(swap_avail_raw) - sused = stotal - savail + sused = stotal - savail metrics["swap_total_bytes"] = stotal * 1024 - metrics["swap_used_bytes"] = sused * 1024 - metrics["swap_percent"] = round(sused / stotal * 100, 1) if stotal else 0.0 + metrics["swap_used_bytes"] = sused * 1024 + metrics["swap_percent"] = round(sused / stotal * 100, 1) if stotal else 0.0 except ValueError: pass @@ -178,7 +198,18 @@ async def collect_ucd_metrics( except ValueError: pass - build_if_metrics(metrics, descr, speed, in_oct, out_oct, in_err, out_err, - if_skip=if_skip, tx_err_is_drop=tx_err_is_drop, - hc_in_oct=hc_in_oct, hc_out_oct=hc_out_oct, high_speed=high_speed) + build_if_metrics( + metrics, + descr, + speed, + in_oct, + out_oct, + in_err, + out_err, + if_skip=if_skip, + tx_err_is_drop=tx_err_is_drop, + hc_in_oct=hc_in_oct, + hc_out_oct=hc_out_oct, + high_speed=high_speed, + ) return metrics diff --git a/napalm_device_types/access_point.py b/napalm_device_types/access_point.py index b5dbc9f..2e8a9d6 100644 --- a/napalm_device_types/access_point.py +++ b/napalm_device_types/access_point.py @@ -10,14 +10,12 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.mac_acl import MacAclMixin -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.interface_filter import InterfaceFilterMixin +from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.models import ( Dot1XConfigDict, FastTransitionConfigDict, @@ -26,9 +24,20 @@ from napalm_device_types.models import ( SSIDBridgeDict, WirelessConfigDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.updates import UpdateMixin -class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, InterfaceFilterMixin, DeviceTypeDriver): +class AccessPointDriver( + MacAclMixin, + UpdateMixin, + ServiceControlMixin, + PackageManagementMixin, + HealthMetricsMixin, + InterfaceFilterMixin, + DeviceTypeDriver, +): """ Abstract intermediate driver for wireless access points. @@ -42,11 +51,6 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa ROLE: str = "access_point" TYPE_LABEL: str = "Access Point" - - - - - # AP-specific methods (get_wireless_clients, get_ssids, get_radio_status, # get_interfaces, get_vlans, …) are intentionally NOT defined here. # AccessPointDriver sits AFTER the mixin classes in the MRO of concrete @@ -86,7 +90,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_fast_transition_config(self) -> Dict[str, FastTransitionConfigDict]: + def get_fast_transition_config(self) -> dict[str, FastTransitionConfigDict]: """ Returns the 802.11r Fast BSS Transition (FT) configuration per SSID. @@ -118,7 +122,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_mesh_config(self) -> Dict[str, MeshConfigDict]: + def get_mesh_config(self) -> dict[str, MeshConfigDict]: """ Returns the 802.11s mesh configuration per mesh interface. @@ -148,7 +152,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_mesh_peers(self) -> List[MeshPeerDict]: + def get_mesh_peers(self) -> list[MeshPeerDict]: """ Returns a list of currently active 802.11s mesh peers. @@ -178,7 +182,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - def get_ssid_bridge_config(self) -> Dict[str, SSIDBridgeDict]: + def get_ssid_bridge_config(self) -> dict[str, SSIDBridgeDict]: """ Returns the Layer-2 bridging configuration for each SSID, i.e. which bridge interface and VLAN each SSID is mapped to. @@ -186,7 +190,8 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa Keys are SSID names. Each value contains: * ssid (string) - SSID name (repeated for convenience) - * bridge (string) - bridge interface the VAP is attached to (e.g. ``"br-lan"``, ``"br-guest"``) + * bridge (string) - bridge interface the VAP is attached to + (e.g. ``"br-lan"``, ``"br-guest"``) * vlan_id (int) - 802.1Q VLAN ID (0 = untagged / no VLAN separation) * tagged (bool) - whether traffic is 802.1Q-tagged on the uplink port * client_isolation (bool) - whether clients on this SSID are isolated from each other @@ -219,9 +224,7 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa """ ... - - - def get_dot1x_config(self) -> Dict[str, Dot1XConfigDict]: + def get_dot1x_config(self) -> dict[str, Dot1XConfigDict]: """ Returns the 802.1X / WPA-Enterprise (RADIUS) configuration per SSID. @@ -268,12 +271,3 @@ class AccessPointDriver(MacAclMixin, UpdateMixin, ServiceControlMixin, PackageMa } """ ... - - - - - - - - - diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py new file mode 100644 index 0000000..daa70c3 --- /dev/null +++ b/napalm_device_types/apt_proxy.py @@ -0,0 +1,665 @@ +"""apt's proxy: what apt fetches through, whether a proxy answers, and pointing apt at one. + +Read the same way on every Debian-family host, so the commands and their parse +live here once, like :mod:`~napalm_device_types.host_status`; a driver only +carries a command across and runs one as root. + +**What apt fetches through** is what ``apt-config dump`` says, not one file. apt +reads every file in ``Dir::Etc::parts`` (``apt.conf.d``) in byte order, then +``Dir::Etc::main`` (``apt.conf``); for a single value the last one read wins. The +Debian installer writes its proxy into ``apt.conf``, Ubuntu's into +``apt.conf.d/90curtin-aptproxy``: both win over an ``00proxy`` +(NetOrk/netork#1397). ``DIRECT`` for the proxy, or for one host +(``Acquire::http::Proxy::``), means no proxy there. + +**https** goes through the http proxy too. apt's https method asks +``Acquire::https::Proxy`` first and ``Acquire::http::Proxy`` after it, for one +host and then for all (NetOrk/netork#1811). An http proxy alone therefore takes +every https source as well, and a cache that does not tunnel them answers 403. + +**Where it is set** is every line of apt's files that sets the http proxy, in the +order apt reads them; the https proxy's lines are named apart. A line of the form +``Acquire::http::Proxy "…";`` can be commented out; a setting inside a block +(``Acquire::http { Proxy "…"; };``) is named but left alone. apt skips a file in +``apt.conf.d`` whose name has other characters than letters, digits, ``_``, ``-`` +and ``.``, or an extension other than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); +so is it here. + +**Whether a proxy answers** is a fetch with apt's own downloader, +``apt-helper download-file``, which is on every apt host and needs no root: +straight to a page the caller names (past any proxy), or through the proxy apt is +set to. An HTTP error is an answer too: the other end was reached. Whether the +page is the right one, the caller says (``expect``). + +**Pointing apt at a proxy** writes :attr:`AptProxyMixin.APT_PROXY_FILE`, comments +out every other line that sets a different one, reads the result back from +``apt-config``, and fetches the ``InRelease`` of one http source through it. The +last step is what proves it: apt reached a mirror through the proxy it is now set +to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one +https source the same way; where the proxy refuses it, https goes ``DIRECT``. +""" + +from __future__ import annotations + +import re +import shlex +from collections.abc import Callable +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepOutcomeDict +from napalm_device_types.terminal import strip_terminal_codes + + +class AptProxySettingDict(TypedDict): + """One line of apt's configuration that sets a proxy (``AptProxyMixin``).""" + + file: str + line: int # 1-based + value: str + #: True for ``Acquire::http::Proxy "…";`` on a line of its own, which can be + #: commented out; False for a setting inside a block. + editable: bool + + +class AptProxyReportDict(TypedDict): + """What apt fetches through, and where that is set (``get_package_proxy_report``).""" + + #: The http proxy apt fetches through; None for none or ``DIRECT``. + proxy: str | None + #: ``Acquire::http::Proxy::``: a proxy, or ``DIRECT``, for one host. + per_host: dict[str, str] + #: ``Acquire::http::Proxy-Auto-Detect``: a program that names the proxy. + auto_detect: str | None + #: Every line that sets it, in the order apt reads them; the last one wins. + settings: list[AptProxySettingDict] + #: The proxy apt takes for https: ``Acquire::https::Proxy``, else the http one; + #: None for none or ``DIRECT``. + https_proxy: str | None + #: The per-host proxies https takes: ``Acquire::https::Proxy::``, else + #: ``Acquire::http::Proxy::``. + https_per_host: dict[str, str] + #: Every line that sets ``Acquire::https::Proxy``, in the order apt reads them. + https_settings: list[AptProxySettingDict] + #: The base URIs of the host's sources over http and over https. + http_sources: list[str] + https_sources: list[str] + + +class ProxyFetchDict(TypedDict): + """A fetch from the host with apt's downloader (``check_package_proxy``).""" + + #: Whether the other end answered at all; an HTTP error is an answer. + reachable: bool + #: Whether the page contained what the caller expected; None when it named nothing + #: or nothing came back. + expected: bool | None + #: apt's reason when the fetch failed, else "". + error: str + + +class HttpsSourceFetchDict(ProxyFetchDict): + """One https source fetched the way apt fetches it (``check_https_source``).""" + + #: What was fetched: the source's ``InRelease``. + url: str + #: The proxy apt took for it. + proxy: str + + +_BEGIN = "APROXY_BEGIN" +_END = "APROXY_END" +_APT_HELPER = "/usr/lib/apt/apt-helper" + +#: A proxy URL apt's file and a shell word can both carry. +PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") + +#: One line, POSIX ``sh``, read-only, no privileges. In a subshell, so neither the +#: locale nor its variables stay behind in an interactive session; the frame +#: markers are printed in two halves so that an echoing transport does not show +#: them early. +APT_PROXY_COMMAND = ( + "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " + "echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; " + 'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; ' + "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " + '[ -f "$f" ] && grep -Hin proxy "$f"; done; ' + 'echo \'[main]\'; f="${ap_main:-/etc/apt/apt.conf}"; [ -f "$f" ] && grep -Hin proxy "$f"; ' + "echo '[sources]'; apt-get indextargets --no-release-info --format '$(BASE_URI)' " + "2>/dev/null | sort -u; " + "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" +) + +_DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) +_GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") +_FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) +_NOT_HTTP_PROXY = re.compile( + r"^\s*(//|#)|Acquire::(https|ftp|socks)\b|::Proxy::|Proxy-Auto-Detect", re.I +) +_FLAT_HTTPS = re.compile(r'^\s*Acquire::https::Proxy\s+"([^"]*)"\s*;', re.I) +#: A line that sets the https proxy inside a block, all on the line itself. +_HTTPS_BLOCK = re.compile(r'^\s*Acquire::https\s*\{[^}]*\bProxy\s+"([^"]*)"', re.I) +_IN_BLOCK = re.compile(r'\bProxy\s+"([^"]*)"', re.I) +#: The names apt reads in ``apt.conf.d``: no extension, or ``.conf``. +_PART_NAME = re.compile(r"^(?:[A-Za-z0-9_\-]+|[A-Za-z0-9_.\-]+\.conf)$") +_NO_PROXY = frozenset({"", "direct"}) + + +def _sections(output: str) -> dict[str, list[str]]: + lines = [line.rstrip() for line in strip_terminal_codes(output).splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact apt proxy report in the output") from None + sections: dict[str, list[str]] = {} + current: list[str] = [] + for line, bare in zip(lines[start + 1 : end], stripped[start + 1 : end], strict=True): + if bare.startswith("[") and bare.endswith("]") and " " not in bare: + current = sections.setdefault(bare[1:-1], []) + elif bare: + current.append(line) + return sections + + +def _grep_line(line: str, main: bool) -> tuple[str, int, str] | None: + """File, line number and text of one line grep printed; None for a file apt skips.""" + found = _GREP.match(line) + if not found: + return None + path, number, text = found.groups() + if not main and not _PART_NAME.match(path.rsplit("/", 1)[-1]): + return None + return path, int(number), text + + +def _setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found + flat = _FLAT.match(text) + if flat: + return {"file": path, "line": int(number), "value": flat.group(1), "editable": True} + if _NOT_HTTP_PROXY.search(text): + return None + nested = _IN_BLOCK.search(text) + if not nested: + return None + return {"file": path, "line": int(number), "value": nested.group(1), "editable": False} + + +def _https_setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found + flat = _FLAT_HTTPS.match(text) + if flat: + return {"file": path, "line": number, "value": flat.group(1), "editable": True} + block = _HTTPS_BLOCK.match(text) + if block: + return {"file": path, "line": number, "value": block.group(1), "editable": False} + return None + + +class _Dump(TypedDict): + proxy: str | None + per_host: dict[str, str] + auto_detect: str | None + #: ``Acquire::https::Proxy`` as written; None when unset or empty. + https: str | None + https_per_host: dict[str, str] + + +def _dump(lines: list[str]) -> _Dump: + """The proxies, the per-host proxies and the auto-detect program in ``apt-config dump``.""" + dump: _Dump = { + "proxy": None, + "per_host": {}, + "auto_detect": None, + "https": None, + "https_per_host": {}, + } + for found in (_DUMP.match(line.strip()) for line in lines): + if found: + _take(dump, *found.groups()) + return dump + + +def _take(dump: _Dump, scheme: str, suffix: str | None, value: str) -> None: + """One line of the dump into *dump*; a later one of the same key wins, as in apt.""" + https = scheme.lower() == "https" + if not suffix: + _take_for_all(dump, https, value) + elif suffix.lower() == "-auto-detect": + if not https: + dump["auto_detect"] = value or None + elif value: + dump["https_per_host" if https else "per_host"][suffix[2:]] = value + + +def _take_for_all(dump: _Dump, https: bool, value: str) -> None: + """The proxy for every host: https keeps an empty value as unset, apt's fallback.""" + if https: + dump["https"] = value or None + else: + dump["proxy"] = None if value.lower() in _NO_PROXY else value + + +def _https_proxy(dump: _Dump) -> str | None: + """apt's https method takes its own setting and falls back to the http one.""" + if dump["https"] is None: + return dump["proxy"] + return None if dump["https"].lower() in _NO_PROXY else dump["https"] + + +def https_proxy_for(report: AptProxyReportDict, host: str) -> str | None: + """The proxy apt takes for an https source on *host*; None for ``DIRECT`` or none. + + The host's own setting first, https before http, then the one for all hosts. + """ + for per_host in (report["https_per_host"], report["per_host"]): + value = next((v for h, v in per_host.items() if h.lower() == host.lower()), None) + if value: + return None if value.lower() in _NO_PROXY else value + return report["https_proxy"] + + +def parse_apt_proxy(output: str) -> AptProxyReportDict: + """The report :data:`APT_PROXY_COMMAND` prints, as :class:`AptProxyReportDict`. + + :raises ValueError: if the output carried no intact report. + """ + sections = _sections(output) + dump = _dump(sections.get("dump", [])) + lines = [ + (main, line) + for main, key in ((False, "parts"), (True, "main")) + for line in sections.get(key, []) + ] + sources = [line.strip() for line in sections.get("sources", [])] + return { + "proxy": dump["proxy"], + "per_host": dump["per_host"], + "auto_detect": dump["auto_detect"], + "settings": _settings(lines, _setting), + "https_proxy": _https_proxy(dump), + "https_per_host": {**dump["per_host"], **dump["https_per_host"]}, + "https_settings": _settings(lines, _https_setting), + "http_sources": _of_scheme(sources, "http"), + "https_sources": _of_scheme(sources, "https"), + } + + +def _settings( + lines: list[tuple[bool, str]], parse: Callable[[str, bool], AptProxySettingDict | None] +) -> list[AptProxySettingDict]: + return [s for s in (parse(line, main) for main, line in lines) if s] + + +def _of_scheme(sources: list[str], scheme: str) -> list[str]: + return [s for s in sources if s.startswith(f"{scheme}://")] + + +_DEFAULT_PORT = {"http": 80, "https": 443} + + +def _endpoint(url: str | None) -> tuple[str, str, int | None, str] | None: + """Scheme, host, port (the scheme's own when none is given) and path; None for no URL.""" + if not url: + return None + try: + parts = urlsplit(url) + port = parts.port or _DEFAULT_PORT.get(parts.scheme) + except ValueError: + return None + return parts.scheme, parts.hostname or "", port, parts.path.rstrip("/") + + +def same_proxy_url(a: str | None, b: str | None) -> bool: + """Whether *a* and *b* name the same proxy: scheme, host and port, a trailing ``/`` aside.""" + first = _endpoint(a) + return first is not None and first == _endpoint(b) + + +def apt_fetch_command(url: str, *, direct: bool = False, expect: str = "") -> str: + """The command that fetches *url* with apt's downloader into a temporary file. + + *direct* goes past any proxy apt is set to, to *url*'s host itself; without it + the fetch takes apt's own way. *expect* is text the page should contain. + + :raises ValueError: if *url* is not an http(s) URL a shell word can carry. + """ + if not PROXY_URL.match(url): + raise ValueError(f"not an http URL: {url!r}") + options = ["-o Acquire::Retries=0", "-o Acquire::http::Timeout=10"] + host = urlsplit(url).hostname or "" + if direct and host and ":" not in host: + # *host* comes from a URL PROXY_URL let through: no quote in it. For https + # the host's https setting comes first, so both are set. + if url.startswith("https://"): + options.append(f"-o 'Acquire::https::Proxy::{host}=DIRECT'") + options.append(f"-o 'Acquire::http::Proxy::{host}=DIRECT'") + match = f"echo '[match]'; grep -c -F -- {shlex.quote(expect)} \"$t\"; " if expect else "" + return ( + "( export LC_ALL=C; t=$(mktemp) || exit 1; printf '%s%s\\n' APROXY_ BEGIN; " + f"echo '[fetch]'; {_APT_HELPER} {' '.join(options)} download-file " + f'{shlex.quote(url)} "$t" 2>&1; rc=$?; echo \'[rc]\'; echo "$rc"; ' + f"{match}rm -f \"$t\"; printf '%s%s\\n' APROXY_ END ) | cat" + ) + + +_FAILED = re.compile(r"^E: Failed to fetch \S+\s+(.*)$") +_HTTP_STATUS = re.compile(r"^\d{3}\s") + + +def parse_apt_fetch(output: str) -> ProxyFetchDict: + """What :func:`apt_fetch_command` printed, as :class:`ProxyFetchDict`. + + :raises ValueError: if the output carried no intact report, or apt-helper is + missing -- then nothing could be tried. + """ + sections = _sections(output) + try: + rc = int(sections["rc"][0]) + except (KeyError, IndexError, ValueError): + raise ValueError("no exit status in the apt fetch report") from None + if rc in (126, 127): + raise ValueError(f"{_APT_HELPER} cannot run on this host") + match = sections.get("match") + if rc == 0: + expected = (match[0].strip() not in ("", "0")) if match else None + return {"reachable": True, "expected": expected, "error": ""} + error = next( + (m.group(1).strip() for m in map(_FAILED.match, sections.get("fetch", [])) if m), + "the download failed", + ) + answered = bool(_HTTP_STATUS.match(error)) + return {"reachable": answered, "expected": False if answered else None, "error": error} + + +#: sed's edit of one line: comment it out, if it still sets the proxy of *scheme*. +_COMMENT_OUT = "-e '{line}{{/^[[:space:]]*Acquire::{scheme}::Proxy[[:space:]]/Is|^|// |}}'" +_SCHEMES = ("http", "https") + + +def comment_out_command(path: str, lines: list[int], scheme: str = "http") -> str: + """The root command that comments out *lines* of *path* where they set *scheme*'s proxy.""" + if scheme not in _SCHEMES: + raise ValueError(f"not a scheme apt proxies: {scheme!r}") + edits = " ".join(_COMMENT_OUT.format(line=int(n), scheme=scheme) for n in lines) + return f"sed -i {edits} {shlex.quote(path)}" + + +def _editable_lines(settings: list[AptProxySettingDict]) -> dict[str, list[int]]: + by_file: dict[str, list[int]] = {} + for s in settings: + if s["editable"]: + by_file.setdefault(s["file"], []).append(s["line"]) + return by_file + + +def _described(s: AptProxySettingDict) -> str: + what = "commented out" if s["editable"] else "left alone (inside a block)" + return f"{s['file']}:{s['line']} ({s['value']}) {what}" + + +class AptProxyMixin(PackageProxyMixin): + """:class:`~napalm_device_types.fixes.PackageProxyMixin` for a Debian-family host. + + The template form, like :class:`~napalm_device_types.host_status.HostStatusMixin`: + the reading, the parse and the order of the fix are the same everywhere; a + driver supplies the transports below. Mixed in by the drivers that can, so + ``callable(getattr(driver, "configure_package_proxy", None))`` stays a truthful + answer. + """ + + #: The file the proxy is written to. Its name carries no application's. + APT_PROXY_FILE = "/etc/apt/apt.conf.d/00proxy" + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_apt_command(self, command: str) -> str: + """Run *command* on the host with ``sh``, without privileges; what it printed.""" + ... + + def _run_apt_root_command(self, command: str, done: str) -> FixStepOutcomeDict: + """Run *command* as root; ``ok`` with *done* when it exited 0.""" + ... + + def _write_apt_file(self, path: str, content: str) -> FixStepOutcomeDict: + """Replace *path* with *content*, as a new file owned by root, mode 644.""" + ... + + def _is_apt_host(self) -> bool: + """Whether apt is the host's package manager; a driver that can tell overrides it.""" + return True + + def get_package_proxy_report(self) -> AptProxyReportDict | None: + """What apt fetches through and where that is set; None on a host without apt. + + * proxy (string or None): the http proxy apt fetches through + * per_host ({host: proxy or "DIRECT"}) + * auto_detect (string or None): ``Acquire::http::Proxy-Auto-Detect`` + * settings (list of {file, line, value, editable}), in apt's reading order + * https_proxy (string or None): the proxy https takes, the http one if unset + * https_per_host ({host: proxy or "DIRECT"}), https_settings (as settings) + * http_sources, https_sources (lists of the sources' base URIs) + + :raises ValueError: if the host's output carried no intact report. + """ + if not self._is_apt_host(): + return None + return parse_apt_proxy(self._run_apt_command(APT_PROXY_COMMAND)) + + def get_package_proxy(self) -> str | None: + """The proxy apt fetches through now; None for none, or a host without apt.""" + report = self.get_package_proxy_report() + return report["proxy"] if report else None + + def check_package_proxy(self, url: str, expect: str = "") -> ProxyFetchDict: + """Fetch *url* from the host, past any proxy apt is set to. + + Whether the host reaches it, and whether the page contains *expect*. + + :raises ValueError: if *url* is no http URL, or the host cannot fetch at all. + :raises NotImplementedError: on a host without apt. + """ + if not self._is_apt_host(): + raise NotImplementedError("not an apt host") + return parse_apt_fetch( + self._run_apt_command(apt_fetch_command(url, direct=True, expect=expect)) + ) + + def check_https_source(self) -> HttpsSourceFetchDict | None: + """Fetch one https source's ``InRelease`` the way apt does, through its proxy. + + None when no https source goes through a proxy: there are none, or https + goes ``DIRECT``. + + :raises ValueError: if the host's output carried no intact report. + :raises NotImplementedError: on a host without apt. + """ + if not self._is_apt_host(): + raise NotImplementedError("not an apt host") + report = self._report() + found = _https_source(report, lambda proxy: proxy is not None) + if found is None: + return None + target, proxy = found + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + return {**fetched, "url": target, "proxy": proxy} + + def configure_package_proxy(self, url: str) -> FixResultDict: + """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`. + + https goes through *url* too, unless *url* refuses it: then ``DIRECT``. + """ + if not url: + return run_fix_steps([("configure", self._remove_package_proxy)]) + return run_fix_steps( + [ + ("configure", lambda: self._write_package_proxy(url)), + ("conflicts", lambda: self._comment_out_other_proxies(url)), + ("verify", lambda: self._verify_package_proxy(url)), + ("fetch", lambda: self._fetch_through_package_proxy(url)), + ("https", lambda: self._https_through_package_proxy(url)), + ] + ) + + def _not_apt(self) -> FixStepOutcomeDict: + return {"ok": False, "output": "not an apt host"} + + def _remove_package_proxy(self) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + return self._run_apt_root_command( + f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" + ) + + def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + if not PROXY_URL.match(url): + raise ValueError(f"not a proxy URL: {url!r}") + content = f'Acquire::http::Proxy "{url}";\n' + if https_direct: + content += 'Acquire::https::Proxy "DIRECT";\n' + return self._write_apt_file(self.APT_PROXY_FILE, content) + + def _report(self) -> AptProxyReportDict: + # The steps after "configure" run only once it passed, on an apt host. + return parse_apt_proxy(self._run_apt_command(APT_PROXY_COMMAND)) + + def _comment_out_other_proxies(self, url: str) -> FixStepOutcomeDict: + others = [ + s + for s in self._report()["settings"] + if s["file"] != self.APT_PROXY_FILE and not same_proxy_url(s["value"], url) + ] + for path, numbers in _editable_lines(others).items(): + outcome = self._run_apt_root_command( + comment_out_command(path, numbers), f"commented out in {path}" + ) + if not outcome["ok"]: + return outcome + return {"ok": True, "output": "; ".join(map(_described, others)) or "no other proxy set"} + + def _verify_package_proxy(self, url: str) -> FixStepOutcomeDict: + report = self._report() + if same_proxy_url(report["proxy"], url): + return {"ok": True, "output": f"apt fetches through {url}"} + where = [ + f"{s['file']}:{s['line']}" + for s in report["settings"] + if s["value"] == report["proxy"] and s["file"] != self.APT_PROXY_FILE + ] + return { + "ok": False, + "output": f"apt still fetches through {report['proxy'] or 'no proxy'}" + + (f", set in {', '.join(where)}" if where else ""), + } + + def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict: + report = self._report() + direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} + source = next( + ( + s + for s in report["http_sources"] + if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct + ), + None, + ) + if source is None: + return {"ok": True, "output": "no http source to fetch through the proxy; not tried"} + target = source.rstrip("/") + "/InRelease" + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if fetched["reachable"] and not fetched["error"]: + return {"ok": True, "output": f"fetched {target} through {url}"} + return {"ok": False, "output": f"{target}: {fetched['error']}"} + + def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict: + """An https source through *url*; where *url* refuses it, https goes ``DIRECT``.""" + found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url)) + if found is None: + return {"ok": True, "output": f"no https source goes through {url}; not tried"} + target = found[0] + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + # An HTTP error came from the mirror, through the tunnel: the cache let it by. + if fetched["reachable"]: + return {"ok": True, "output": f"fetched {target} through {url}"} + refused = f"{url} refused https ({fetched['error']})" + outcome = self._https_direct(url) + if not outcome["ok"]: + return {"ok": False, "output": f"{refused}; {outcome['output']}"} + direct = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if not direct["reachable"]: + return {"ok": False, "output": f"{refused}; direct too: {target}: {direct['error']}"} + named = f"; {outcome['output']}" if outcome["output"] else "" + return { + "ok": True, + "output": f"{refused}{named}; https sources go direct now: fetched {target}", + } + + def _https_direct(self, url: str) -> FixStepOutcomeDict: + """Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*.""" + written = self._write_package_proxy(url, https_direct=True) + if not written["ok"]: + return written + later = self._later_https_lines(self._report(), url) + for path, numbers in _editable_lines(later).items(): + outcome = self._run_apt_root_command( + comment_out_command(path, numbers, scheme="https"), f"commented out in {path}" + ) + if not outcome["ok"]: + return outcome + report = self._report() + if same_proxy_url(report["https_proxy"], url): + still = [f"{s['file']}:{s['line']}" for s in self._later_https_lines(report, url)] + where = f", set in {', '.join(still)}" if still else "" + return {"ok": False, "output": f"https still goes through {url}{where}"} + return {"ok": True, "output": "; ".join(map(_described, later))} + + def _later_https_lines(self, report: AptProxyReportDict, url: str) -> list[AptProxySettingDict]: + """The lines outside :attr:`APT_PROXY_FILE` that send https to *url*.""" + return [ + s + for s in report["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + + +def _https_source( + report: AptProxyReportDict, through: Callable[[str | None], bool] +) -> tuple[str, str] | None: + """The ``InRelease`` of the first https source whose proxy *through* accepts, and that proxy.""" + for source in report["https_sources"]: + if "/dists/" not in source: + continue + proxy = https_proxy_for(report, urlsplit(source).hostname or "") + if through(proxy): + return source.rstrip("/") + "/InRelease", proxy or "" + return None + + +__all__ = [ + "APT_PROXY_COMMAND", + "AptProxyMixin", + "AptProxyReportDict", + "AptProxySettingDict", + "HttpsSourceFetchDict", + "ProxyFetchDict", + "PROXY_URL", + "apt_fetch_command", + "comment_out_command", + "https_proxy_for", + "parse_apt_fetch", + "parse_apt_proxy", + "same_proxy_url", +] diff --git a/napalm_device_types/arp.py b/napalm_device_types/arp.py new file mode 100644 index 0000000..c543a89 --- /dev/null +++ b/napalm_device_types/arp.py @@ -0,0 +1,13 @@ +"""The ``age`` of an entry in NAPALM's neighbour tables when the device does not say. + +NAPALM requires an ``age`` on every entry of ``get_arp_table`` and +``get_ipv6_neighbors_table``: seconds since the device last confirmed that +neighbour. Many devices do not report one. ``0.0`` means "confirmed just now" +and must never stand in for "not reported": a caller ranking sightings by +freshness takes it at its word (NetOrk/netork#1327). An expiry countdown is not +an age either. The value is NAPALM's own; its IOS driver reports ``-1.0``. +""" + +from typing import Final + +ARP_AGE_UNKNOWN: Final[float] = -1.0 diff --git a/napalm_device_types/base.py b/napalm_device_types/base.py index 97f0f9b..b95622f 100644 --- a/napalm_device_types/base.py +++ b/napalm_device_types/base.py @@ -15,6 +15,7 @@ from napalm.base import NetworkDriver from napalm_device_types.channel import CommandChannelMixin from napalm_device_types.host_reboot import HostRebootMixin +from napalm_device_types.host_shutdown import HostShutdownMixin from napalm_device_types.ping_sweep import PingSweepMixin @@ -74,7 +75,13 @@ class PortSpec(NamedTuple): mandatory: bool = False -class DeviceTypeDriver(PingSweepMixin, HostRebootMixin, CommandChannelMixin, NetworkDriver): # type: ignore[misc] +class DeviceTypeDriver( + PingSweepMixin, + HostRebootMixin, + HostShutdownMixin, + CommandChannelMixin, + NetworkDriver, # type: ignore[misc] +): """Common base for all netOrk device-type drivers. Sits between napalm.base.NetworkDriver and the type-specific abstract diff --git a/napalm_device_types/channel.py b/napalm_device_types/channel.py index 6b534c9..775e458 100644 --- a/napalm_device_types/channel.py +++ b/napalm_device_types/channel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The command channel: a public way to run a command on a host, or open a stream. A driver already reaches its host, over SSH, a REST API or WinRM. What it did @@ -26,9 +25,8 @@ stays with the driver. from __future__ import annotations -import socket import time -from typing import Any, NamedTuple, Optional, Protocol, Tuple, TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, NamedTuple, Protocol, cast #: How much of a stream's stderr is kept. Enough to tell "permission denied" #: from "no such command"; a stream that writes megabytes there keeps its tail. @@ -56,7 +54,7 @@ class ByteStream(Protocol): ``close_write`` closes the writing side. """ - def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes: ... + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: ... def write(self, data: bytes) -> None: ... @@ -65,7 +63,7 @@ class ByteStream(Protocol): def close(self) -> None: ... @property - def exit_status(self) -> Optional[int]: ... + def exit_status(self) -> int | None: ... @property def stderr(self) -> str: ... @@ -82,7 +80,7 @@ class CommandChannelMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: """Run *command* with the host's shell; *stdin*, if given, is written first.""" ... @@ -108,11 +106,11 @@ class ParamikoExecStream: while self._channel.recv_stderr_ready(): self._stderr = (self._stderr + self._channel.recv_stderr(_READ))[-self._limit :] - def read(self, max_bytes: int, timeout: Optional[float] = None) -> bytes: + def read(self, max_bytes: int, timeout: float | None = None) -> bytes: self._channel.settimeout(timeout) try: return cast(bytes, self._channel.recv(max_bytes)) - except socket.timeout: + except TimeoutError: raise TimeoutError(f"no data within {timeout}s") from None finally: self._drain_stderr() @@ -127,7 +125,7 @@ class ParamikoExecStream: self._channel.close() @property - def exit_status(self) -> Optional[int]: + def exit_status(self) -> int | None: if not self._channel.exit_status_ready(): return None return cast(int, self._channel.recv_exit_status()) @@ -138,7 +136,7 @@ class ParamikoExecStream: return self._stderr.decode("utf-8", "replace") -def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: +def _collect(channel: Any, command: str, timeout: float) -> tuple[bytes, bytes]: out, err = b"", b"" deadline = time.monotonic() + timeout while True: @@ -153,7 +151,7 @@ def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: out += chunk while chunk := channel.recv_stderr(_READ): err += chunk - except socket.timeout: + except TimeoutError: raise TimeoutError(f"{command!r} did not finish within {timeout}s") from None return out, err elif time.monotonic() > deadline: @@ -163,7 +161,7 @@ def _collect(channel: Any, command: str, timeout: float) -> Tuple[bytes, bytes]: def run_on_transport( - transport: Any, command: str, *, stdin: Optional[bytes] = None, timeout: float = 60 + transport: Any, command: str, *, stdin: bytes | None = None, timeout: float = 60 ) -> CommandResult: """Run *command* on an exec channel of the paramiko *transport*.""" channel = transport.open_session() @@ -175,14 +173,16 @@ def run_on_transport( channel.shutdown_write() out, err = _collect(channel, command, timeout) return CommandResult( - out.decode("utf-8", "replace"), err.decode("utf-8", "replace"), channel.recv_exit_status() + out.decode("utf-8", "replace"), + err.decode("utf-8", "replace"), + channel.recv_exit_status(), ) finally: channel.close() def open_stream_on_transport( - transport: Any, command: str, *, stdin_prefix: Optional[bytes] = None + transport: Any, command: str, *, stdin_prefix: bytes | None = None ) -> ParamikoExecStream: """Start *command* on an exec channel and return its stream. diff --git a/napalm_device_types/clusters.py b/napalm_device_types/clusters.py new file mode 100644 index 0000000..cbc77ab --- /dev/null +++ b/napalm_device_types/clusters.py @@ -0,0 +1,99 @@ +"""The shapes of a hypervisor's clusters (``HypervisorDriver.get_clusters()``). + +Apart from ``models.py``, which is over the file size limit. +""" + +from typing import NotRequired, TypedDict + + +class ClusterNodeDict(TypedDict): + """One host of a cluster, as its cluster sees it. A value the platform does not + report is None; it is never guessed.""" + + name: str # the node's name in its cluster, as get_vms() reports it in ``node`` + status: str # "online", "offline" or "unknown" + local: bool # the node this connection talks to; False on a manager (vCenter) + maintenance: bool | None # in maintenance mode + cpu_threads: int | None # logical CPUs + cpu_cores: int | None # physical cores + cpu_model: str | None + cpu_flags: list[str] | None # cpuinfo flags; None where the platform does not say + cpu_usage: float | None # 0.0–1.0 + memory_total: int | None # bytes + memory_used: int | None # bytes + votes: NotRequired[int | None] # quorum votes of this node; None standalone/unreadable + + +class ClusterStorageDict(TypedDict): + """A storage of a cluster and the nodes that see it.""" + + name: str # usable as create_vm_from_cloud_init(storage=...) on each of ``nodes`` + type: str # backend type: "dir", "lvmthin", "zfspool", "nfs", "rbd", "vmfs", ... + shared: bool # the same storage on every node in ``nodes``, so a VM on it can move + content: list[str] # what it holds: "images", "iso", "snippets", ... + nodes: list[str] # the nodes it is available on + total: int | None # bytes; None where the platform does not report it + used: int | None # bytes + + +class ClusterNetworkDict(TypedDict): + """A network a VM's NIC may attach to, and the nodes that carry it.""" + + name: str # usable as NICConfigDict.bridge on each of ``nodes`` + kind: str # "bridge", "vnet" or "portgroup", as in NetworkTargetDict + vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of it + fixed_vlan_tag: NotRequired[int | None] # vnet/portgroup: the VLAN baked into it + nodes: list[str] + + +class HAResourceDict(TypedDict): + """A guest the cluster's HA manager looks after.""" + + vmid: int + kind: str # "qemu" or "lxc" + requested: str # state HA is asked to keep: "started", "stopped", "disabled", "ignored" + status: str | None # what the HA manager reports now: "started", "fence", "migrate", ... + node: str | None # the node the HA manager places it on now + group: NotRequired[str | None] # HA group (Proxmox VE 8) + rules: list[str] # ids of the HA rules that name it (Proxmox VE 9); [] where there are none + + +class BackupJobDict(TypedDict): + """A scheduled backup job of the cluster.""" + + id: str + enabled: bool + schedule: str | None # as the platform writes it: "sat 02:00", "daily", "mon..fri 21:00" + storage: str | None + mode: str | None # "snapshot", "suspend", "stop" + all: bool # every guest, minus ``exclude`` + vmids: list[int] # the guests it names; a pool job: the pool's members now; [] with all + exclude: list[int] + pool: str | None + node: str | None # the one node it runs on; None for every node + + +class BackupDict(TypedDict): + """The backups of one guest on one storage.""" + + vmid: int + storage: str + time: int # unix seconds of the newest backup of this guest on this storage + size: int | None # bytes of that newest backup + count: int # backups of this guest on this storage + + +class ClusterDict(TypedDict): + """A cluster the connection can see. A standalone host is a cluster of one, + with no name.""" + + id: str | None # stable across connections where the platform has one; None otherwise + name: str | None # the cluster's name; None for a standalone host + quorate: bool | None # None where the platform has no quorum (standalone, vSphere) + nodes: list[ClusterNodeDict] + storages: list[ClusterStorageDict] + networks: list[ClusterNetworkDict] + # None: could not be read; []: read, nothing there. + ha_resources: NotRequired[list[HAResourceDict] | None] + backup_jobs: NotRequired[list[BackupJobDict] | None] + backups: NotRequired[list[BackupDict] | None] diff --git a/napalm_device_types/config_lifecycle.py b/napalm_device_types/config_lifecycle.py index a27b5dd..c205bda 100644 --- a/napalm_device_types/config_lifecycle.py +++ b/napalm_device_types/config_lifecycle.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- from __future__ import annotations import difflib -from typing import ClassVar, TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from napalm.base.exceptions import ( CommandErrorException, @@ -41,15 +40,11 @@ class ConfigLifecycleMixin: if TYPE_CHECKING: - def _get_running_config(self) -> str: - ... + def _get_running_config(self) -> str: ... - def commit_config(self, message: str = "", revert_in: int | None = None) -> None: - ... + def commit_config(self, message: str = "", revert_in: int | None = None) -> None: ... - def load_merge_candidate( - self, filename: str | None = None, config: str | None = None - ) -> None: + def load_merge_candidate(self, filename: str | None = None, config: str | None = None) -> None: if filename is not None: try: with open(filename) as fh: diff --git a/napalm_device_types/config_sync.py b/napalm_device_types/config_sync.py new file mode 100644 index 0000000..7990cb8 --- /dev/null +++ b/napalm_device_types/config_sync.py @@ -0,0 +1,34 @@ +"""Configuration sync between the two hosts of an HA pair. + +A host that pushes its configuration to a peer (OPNsense's XMLRPC sync, say) +makes the peer's configuration of the synced services a copy of its own: a +change written on the receiving side lasts only until the next push. Who pushes +what is therefore a fact a caller needs before it writes to either host. + +Declared under ``if TYPE_CHECKING``: a contract, not a placeholder. A driver +mixes it in; no role promises it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import ConfigSyncDict + + +class ConfigSyncMixin: + """Where a host pushes its configuration, and for which of its services.""" + + if TYPE_CHECKING: + + def get_config_sync(self) -> ConfigSyncDict: + """ + Returns the peer this host pushes its configuration to (None when it + pushes to no one) and the services whose configuration it pushes, by + the name ``get_services`` reports. Sync items the driver has no + service name for are listed under ``unmapped`` as the host names them. + + :raises Exception: When the setting could not be read -- never + ``peer=None`` for "could not ask". + """ + ... diff --git a/napalm_device_types/container_engine.py b/napalm_device_types/container_engine.py index 265940a..f0e35f7 100644 --- a/napalm_device_types/container_engine.py +++ b/napalm_device_types/container_engine.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Access to a host's container engine, and nothing more. This package abstracts the connection, the driver builds it, and netOrk does @@ -29,13 +28,14 @@ stays truthful. from __future__ import annotations import shlex -from typing import Any, Dict, List, Optional, Sequence, TYPE_CHECKING, cast +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, cast from napalm_device_types.channel import ByteStream, CommandResult -from napalm_device_types.models import ContainerEngineDict +from napalm_device_types.models import ContainerEngineDict, FixResultDict #: Engines this package knows how to reach, and the API each speaks. -ENGINES: Dict[str, str] = {"docker": "docker-engine"} +ENGINES: dict[str, str] = {"docker": "docker-engine"} _PROBE_TIMEOUT = 15 _CLI_TIMEOUT = 120 @@ -68,7 +68,7 @@ class ContainerEngineConnection: self, args: Sequence[str], *, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, timeout: float = _CLI_TIMEOUT, privileged: bool = False, ) -> CommandResult: @@ -77,9 +77,12 @@ class ContainerEngineConnection: *privileged* runs it as root, the way the driver gains root for any command: for a call the engine refused to the login user. """ - return cast(CommandResult, self._driver.run_command( - self._command(args), privileged=privileged, timeout=timeout, stdin=stdin - )) + return cast( + CommandResult, + self._driver.run_command( + self._command(args), privileged=privileged, timeout=timeout, stdin=stdin + ), + ) def stream_cli( self, args: Sequence[str], *, merge_stderr: bool = False, privileged: bool = False @@ -90,9 +93,12 @@ class ContainerEngineConnection: progress there (``compose up``); *privileged* as for :meth:`run_cli`. """ command = self._command(args) - return cast(ByteStream, self._driver.open_stream( - f"{command} 2>&1" if merge_stderr else command, privileged=privileged - )) + return cast( + ByteStream, + self._driver.open_stream( + f"{command} 2>&1" if merge_stderr else command, privileged=privileged + ), + ) class ContainerEngineMixin: @@ -106,16 +112,22 @@ class ContainerEngineMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: ... def open_stream(self, command: str, *, privileged: bool = False) -> ByteStream: ... + def grant_container_engine_access(self, user: str) -> FixResultDict: + """Let *user* reach the container engine without privilege (the + ``docker`` group on most hosts). A fix template: see + :mod:`napalm_device_types.fixes`. Declared only; a driver implements it.""" + ... + def _container_engine_binary(self, engine: str) -> str: """The host command that is *engine*'s CLI. Override where it is not on PATH.""" return engine - def container_engines(self) -> List[ContainerEngineDict]: + def container_engines(self) -> list[ContainerEngineDict]: """ Returns the container engines the host offers: @@ -126,7 +138,7 @@ class ContainerEngineMixin: user may use it is decided by the first API call, which says "permission denied" in the stream's stderr. """ - found: List[ContainerEngineDict] = [] + found: list[ContainerEngineDict] = [] for engine, api in ENGINES.items(): probe = shlex.join(["command", "-v", self._container_engine_binary(engine)]) result = self.run_command(probe, timeout=_PROBE_TIMEOUT) diff --git a/napalm_device_types/dhcp.py b/napalm_device_types/dhcp.py index 0992c63..f1f72ef 100644 --- a/napalm_device_types/dhcp.py +++ b/napalm_device_types/dhcp.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """DHCP desired-state management, shared by firewalls and gateways. Covers two independent desired-state sets: @@ -24,15 +23,17 @@ takes DHCP down for an entire VLAN. from __future__ import annotations import re -from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( - DhcpReservationDiffDict, + DhcpHaDict, DhcpReservationDict, + DhcpReservationDiffDict, DhcpReservationUpdateDict, - DhcpSubnetDiffDict, DhcpSubnetDict, + DhcpSubnetDiffDict, DhcpSubnetUpdateDict, ) @@ -57,7 +58,7 @@ _DHCP_SUBNET_COMPARE_FIELDS = ( _HEX_ONLY = re.compile(r"[^0-9a-f]") -def normalize_mac(mac: Optional[str]) -> str: +def normalize_mac(mac: str | None) -> str: """Reduces a MAC address to lowercase colon-separated form. Devices report MACs in whatever form their config store happens to use -- @@ -79,7 +80,7 @@ def normalize_mac(mac: Optional[str]) -> str: return ":".join(hex_digits[i : i + 2] for i in range(0, 12, 2)) -def normalize_cidr(cidr: Optional[str]) -> str: +def normalize_cidr(cidr: str | None) -> str: """Reduces a subnet CIDR to a canonical string for matching. Only whitespace and case are normalised -- deliberately not the network @@ -90,9 +91,7 @@ def normalize_cidr(cidr: Optional[str]) -> str: return (cidr or "").strip().lower() -def _subnet_field_differs( - field: str, live: DhcpSubnetDict, desired: DhcpSubnetDict -) -> bool: +def _subnet_field_differs(field: str, live: DhcpSubnetDict, desired: DhcpSubnetDict) -> bool: """Compares one subnet field, with `option_data` handled specially. For `option_data` only the options `desired` actually names are compared; @@ -104,9 +103,7 @@ def _subnet_field_differs( desired_options = desired.get("option_data") or {} live_options = live.get("option_data") or {} - return any( - live_options.get(option) != value for option, value in desired_options.items() - ) + return any(live_options.get(option) != value for option, value in desired_options.items()) class DhcpServerMixin: @@ -124,7 +121,7 @@ class DhcpServerMixin: if TYPE_CHECKING: - def get_dhcp_reservations(self) -> List[DhcpReservationDict]: + def get_dhcp_reservations(self) -> list[DhcpReservationDict]: """ Returns all static DHCP reservations currently configured on the device, across all subnets. @@ -138,8 +135,8 @@ class DhcpServerMixin: ... def apply_dhcp_reservation( - self, reservation: DhcpReservationDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single static DHCP reservation on the device. @@ -156,7 +153,7 @@ class DhcpServerMixin: """ ... - def get_dhcp_subnets(self) -> List[DhcpSubnetDict]: + def get_dhcp_subnets(self) -> list[DhcpSubnetDict]: """ Returns every DHCPv4 subnet the device serves, with its pools and per-subnet options. @@ -167,8 +164,8 @@ class DhcpServerMixin: ... def apply_dhcp_subnet( - self, subnet: DhcpSubnetDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, subnet: DhcpSubnetDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single DHCPv4 subnet on the device. @@ -188,7 +185,7 @@ class DhcpServerMixin: """ ... - def commit_dhcp_subnets(self) -> Dict[str, Any]: + def commit_dhcp_subnets(self) -> dict[str, Any]: """ Applies pending subnet changes (e.g. Kea's ``service/reconfigure``). @@ -203,7 +200,7 @@ class DhcpServerMixin: """ ... - def commit_dhcp_reservations(self) -> Dict[str, Any]: + def commit_dhcp_reservations(self) -> dict[str, Any]: """ Applies pending reservation changes (e.g. Kea's ``service/reconfigure``, or a dnsmasq reload). @@ -219,13 +216,25 @@ class DhcpServerMixin: """ ... + def get_dhcp_ha(self) -> DhcpHaDict: + """ + Returns what the server's own HA configuration declares: whether HA + is switched on, which peer this server is, and each peer with its + role as the server words it. + + Optional: a driver without it has no ``get_dhcp_ha`` at all, and a + caller asks ``callable(getattr(driver, "get_dhcp_ha", None))``. + + :raises Exception: When the configuration could not be read -- never + ``enabled=False`` for "could not ask". + """ + ... + # ------------------------------------------------------------------ # Generic, vendor-neutral algorithms. # ------------------------------------------------------------------ - def diff_dhcp_reservations( - self, desired: List[DhcpReservationDict] - ) -> DhcpReservationDiffDict: + def diff_dhcp_reservations(self, desired: list[DhcpReservationDict]) -> DhcpReservationDiffDict: """ Compares `desired` against the device's current reservations and returns what would need to change to reach that state. @@ -248,14 +257,15 @@ class DhcpServerMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired reservation cannot name a single live one. """ + def key(reservation: DhcpReservationDict) -> str: return normalize_mac(reservation.get("mac")) live_by_mac, shared = index_live(self.get_dhcp_reservations(), key) check_desired(desired, key, shared, "MAC") - add: List[DhcpReservationDict] = [] - update: List[DhcpReservationUpdateDict] = [] + add: list[DhcpReservationDict] = [] + update: list[DhcpReservationUpdateDict] = [] for desired_reservation in desired: live = live_by_mac.get(key(desired_reservation)) @@ -279,9 +289,7 @@ class DhcpServerMixin: return {"add": add, "update": update} - def apply_dhcp_reservationset( - self, desired: List[DhcpReservationDict] - ) -> Iterator[str]: + def apply_dhcp_reservationset(self, desired: list[DhcpReservationDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. @@ -304,8 +312,7 @@ class DhcpServerMixin: self.apply_dhcp_reservation(entry["reservation"], uuid=entry["uuid"]) fields = ", ".join(entry["changed_fields"]) yield ( - f"[update] {entry['reservation']['mac']} -> " - f"{entry['reservation']['ip']} ({fields})" + f"[update] {entry['reservation']['mac']} -> {entry['reservation']['ip']} ({fields})" ) if not diff["add"] and not diff["update"]: @@ -313,12 +320,9 @@ class DhcpServerMixin: return self.commit_dhcp_reservations() - yield ( - f"[commit] applied {len(diff['add'])} add(s), " - f"{len(diff['update'])} update(s)" - ) + yield (f"[commit] applied {len(diff['add'])} add(s), {len(diff['update'])} update(s)") - def diff_dhcp_subnets(self, desired: List[DhcpSubnetDict]) -> DhcpSubnetDiffDict: + def diff_dhcp_subnets(self, desired: list[DhcpSubnetDict]) -> DhcpSubnetDiffDict: """ Compares `desired` against the device's current subnets and returns what would need to change to reach that state. @@ -347,14 +351,15 @@ class DhcpServerMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired subnet cannot name a single live one. """ + def key(subnet: DhcpSubnetDict) -> str: return normalize_cidr(subnet.get("subnet")) live_by_cidr, shared = index_live(self.get_dhcp_subnets(), key) check_desired(desired, key, shared, "subnet") - add: List[DhcpSubnetDict] = [] - update: List[DhcpSubnetUpdateDict] = [] + add: list[DhcpSubnetDict] = [] + update: list[DhcpSubnetUpdateDict] = [] for desired_subnet in desired: live = live_by_cidr.get(key(desired_subnet)) @@ -378,7 +383,7 @@ class DhcpServerMixin: return {"add": add, "update": update} - def apply_dhcp_subnetset(self, desired: List[DhcpSubnetDict]) -> Iterator[str]: + def apply_dhcp_subnetset(self, desired: list[DhcpSubnetDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. @@ -406,7 +411,4 @@ class DhcpServerMixin: return self.commit_dhcp_subnets() - yield ( - f"[commit] applied {len(diff['add'])} add(s), " - f"{len(diff['update'])} update(s)" - ) + yield (f"[commit] applied {len(diff['add'])} add(s), {len(diff['update'])} update(s)") diff --git a/napalm_device_types/dns_forward.py b/napalm_device_types/dns_forward.py new file mode 100644 index 0000000..491c2c5 --- /dev/null +++ b/napalm_device_types/dns_forward.py @@ -0,0 +1,34 @@ +"""Domains a device's DNS resolver forwards to another server. + +An application that makes the device's resolver ask another server for a zone +-- the authority for an internal domain -- edits these. The driver writes them +in its device's terms and applies them; the forwards are the caller's. +Declared only; a driver whose device resolves DNS implements it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import DnsForwardDict + + +class DnsForwardMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_dns_forwards(self) -> list[DnsForwardDict]: + """Every domain the resolver forwards, enabled or not.""" + ... + + def set_dns_forward(self, domain: str, server: str, port: int = 53) -> None: + """Forward *domain* to *server*: added, or the existing forward of + *domain* changed. Applied before it returns. + + :raises RuntimeError: when the device refuses it. + """ + ... + + def remove_dns_forward(self, domain: str) -> None: + """Stop forwarding *domain*; nothing to do when it is not forwarded. + Applied before it returns.""" + ... diff --git a/napalm_device_types/firewall.py b/napalm_device_types/firewall.py index f955898..adbb120 100644 --- a/napalm_device_types/firewall.py +++ b/napalm_device_types/firewall.py @@ -10,21 +10,28 @@ Usage:: ... """ -from typing import Any, ClassVar, Dict, Iterator, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.packages import PackageManagementMixin -from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.dhcp import DhcpServerMixin from napalm_device_types.firewall_rules import FirewallRuleMixin +from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( SecurityZoneDict, SessionDict, ) +from napalm_device_types.nat_vpn import NatVpnMixin +from napalm_device_types.packages import PackageManagementMixin - -class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, FirewallRuleMixin, DhcpServerMixin, DeviceTypeDriver): +class FirewallDriver( + NatVpnMixin, + PackageManagementMixin, + HealthMetricsMixin, + FirewallRuleMixin, + DhcpServerMixin, + DeviceTypeDriver, +): """ Abstract intermediate driver for firewall/security devices. @@ -42,11 +49,9 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi # OPNsense reports drops in the out-error counter. _SNMP_TX_ERR_IS_DROP: ClassVar[bool] = True - if TYPE_CHECKING: - - def get_security_zones(self) -> Dict[str, SecurityZoneDict]: + def get_security_zones(self) -> dict[str, SecurityZoneDict]: """ Returns the security zone configuration. @@ -73,7 +78,7 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - def get_sessions(self) -> List[SessionDict]: + def get_sessions(self) -> list[SessionDict]: """ Returns a list of active connection sessions (stateful flows). @@ -103,8 +108,7 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - - def send_wake_on_lan(self, mac_address: str, interface: str = "") -> Dict[str, Any]: + def send_wake_on_lan(self, mac_address: str, interface: str = "") -> dict[str, Any]: """ Sends a Wake-on-LAN "magic packet" to wake a host on the network. @@ -142,19 +146,9 @@ class FirewallDriver(NatVpnMixin, PackageManagementMixin, HealthMetricsMixin, Fi """ ... - - - - - # ------------------------------------------------------------------ # Firewall rule diff/apply. get_firewall_rules/apply_firewall_rule/ # commit_firewall_rules are abstract (device communication); everything # else here is a concrete, vendor-neutral algorithm -- see README.md # "Design principle: generic vs. device-specific logic". # ------------------------------------------------------------------ - - - - - diff --git a/napalm_device_types/firewall_rules.py b/napalm_device_types/firewall_rules.py index 8858e8f..2324029 100644 --- a/napalm_device_types/firewall_rules.py +++ b/napalm_device_types/firewall_rules.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Firewall rule reconciliation: generic here, device communication in the driver. The split follows the rule in this package's README: matching, diffing and the @@ -14,7 +13,8 @@ working implementation inherited from elsewhere. from __future__ import annotations -from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from napalm_device_types._matching import check_desired, index_live from napalm_device_types.models import ( @@ -43,7 +43,7 @@ class FirewallRuleMixin: if TYPE_CHECKING: - def get_firewall_rules(self) -> List[FirewallRuleDict]: + def get_firewall_rules(self) -> list[FirewallRuleDict]: """ Returns all firewall filter rules currently configured on the device. @@ -55,9 +55,10 @@ class FirewallRuleMixin: firewall rules. """ ... + def apply_firewall_rule( - self, rule: FirewallRuleDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, rule: FirewallRuleDict, *, uuid: str | None = None + ) -> dict[str, Any]: """ Creates or updates a single firewall filter rule on the device. @@ -73,7 +74,8 @@ class FirewallRuleMixin: :returns: A dict with at least ``{"success": bool}``. """ ... - def commit_firewall_rules(self) -> Dict[str, Any]: + + def commit_firewall_rules(self) -> dict[str, Any]: """ Applies pending firewall filter rule changes (e.g. reloads pf/pfctl, or whatever the device's equivalent of "Apply Changes" is). @@ -88,7 +90,7 @@ class FirewallRuleMixin: """ ... - def diff_firewall_rules(self, desired: List[FirewallRuleDict]) -> FirewallRuleDiffDict: + def diff_firewall_rules(self, desired: list[FirewallRuleDict]) -> FirewallRuleDiffDict: """ Compares `desired` against the device's current rules and returns what would need to change to reach that state. @@ -113,14 +115,15 @@ class FirewallRuleMixin: "changed_fields"}, ...]}``. :raises ValueError: When a desired rule cannot name a single live one. """ + def key(rule: FirewallRuleDict) -> str: return (rule.get("description") or "").strip() live_by_description, shared = index_live(self.get_firewall_rules(), key) check_desired(desired, key, shared, "description") - add: List[FirewallRuleDict] = [] - update: List[FirewallRuleUpdateDict] = [] + add: list[FirewallRuleDict] = [] + update: list[FirewallRuleUpdateDict] = [] for desired_rule in desired: live_rule = live_by_description.get(key(desired_rule)) @@ -143,7 +146,8 @@ class FirewallRuleMixin: ) return {"add": add, "update": update} - def apply_firewall_ruleset(self, desired: List[FirewallRuleDict]) -> Iterator[str]: + + def apply_firewall_ruleset(self, desired: list[FirewallRuleDict]) -> Iterator[str]: """ Computes the diff against `desired` and applies it, yielding one human-readable progress line per change, then commits. diff --git a/napalm_device_types/firmware.py b/napalm_device_types/firmware.py new file mode 100644 index 0000000..08b0af1 --- /dev/null +++ b/napalm_device_types/firmware.py @@ -0,0 +1,31 @@ +"""The firmware a vendor offers for a device. + +A fact for the caller (docs/STRUCTURE.md): a driver may know the newer firmware +from the vendor's download catalogue or the device's own update check; whether +running an older one is a problem is decided above the driver. Comparing the +vendor's version strings is vendor knowledge, so the driver says itself whether +the offer is newer. Declared only; a driver that can tell implements it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import FirmwareOfferDict + + +class FirmwareOfferMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_firmware_offer(self) -> FirmwareOfferDict | None: + """The firmware running and the newest the vendor offers. + + None when the driver cannot tell for this device (an unknown model, + a catalogue that did not answer). + + Example:: + + {"current_version": "1.0.2.10", "latest_version": "1.0.2.12", + "newer": True, "download_url": "https://.../GS308E_1.0.2.12.zip"} + """ + ... diff --git a/napalm_device_types/fixes.py b/napalm_device_types/fixes.py new file mode 100644 index 0000000..db1dc2f --- /dev/null +++ b/napalm_device_types/fixes.py @@ -0,0 +1,219 @@ +"""Fix routines: changes an application wants made on a device, as templates. + +A template fixes the order of the steps; the driver knows how each step is done on +its device; the caller supplies every value -- the community, the allowed sources, +the proxy URL, the user. A driver whose device takes the whole change in one call +(a REST API) overrides the template method instead of its steps. + +None of these is mixed into a role base. A driver mixes in the fixes its device can +take, so ``callable(getattr(driver, "configure_snmp_agent", None))`` stays a truthful +answer; a driver that inherits one it cannot run sets it to ``None``. + +They replace ``run_device_action`` (docs/STRUCTURE.md): + +========================== =================================================== +``fix_snmp`` :meth:`SnmpAgentMixin.configure_snmp_agent` +``fix_apt_proxy`` :meth:`PackageProxyMixin.configure_package_proxy` +``fix_docker_permissions`` ``ContainerEngineMixin.grant_container_engine_access`` +``install_lldpd`` and co. :meth:`EnsurePackageMixin.ensure_package` +========================== =================================================== +""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any, ClassVar + +from napalm_device_types.models import ( + FixResultDict, + FixStepDict, + FixStepOutcomeDict, + PackageDict, + SnmpAgentSettingsDict, +) + +#: One step of a fix: its name and the call that does it. +FixStep = tuple[str, Callable[[], FixStepOutcomeDict]] + + +def run_fix_steps(steps: Sequence[FixStep]) -> FixResultDict: + """Run *steps* in order and stop at the first that fails. + + A step that raises has failed, with the exception's message as its output. + """ + done: list[FixStepDict] = [] + for name, step in steps: + try: + outcome = step() + ok, output = bool(outcome["ok"]), str(outcome.get("output", "")) + except Exception as exc: # noqa: BLE001 - any failure of a step ends the fix + ok, output = False, str(exc) + done.append({"step": name, "ok": ok, "output": output}) + if not ok: + break + return { + "success": bool(done) and all(s["ok"] for s in done) and len(done) == len(steps), + "steps": done, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in done), + } + + +class SnmpAgentMixin: + """Run an SNMP agent on the device that answers the caller. + + The steps: install the agent, write its configuration, let the allowed sources + through the host's firewall, start it. Whether it answers is the caller's to ask, + with its own SNMP client. + """ + + def configure_snmp_agent(self, settings: SnmpAgentSettingsDict) -> FixResultDict: + """Install, configure, open and start the agent with *settings*.""" + return run_fix_steps( + [ + ("install", self._install_snmp_agent), + ("configure", lambda: self._write_snmp_agent_config(settings)), + ("firewall", lambda: self._allow_snmp_sources(settings)), + ("start", self._start_snmp_agent), + ] + ) + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_management_peer(self) -> str | None: + """The address the device sees this management connection come from + (for SSH, the client side of ``SSH_CONNECTION``), or None when it cannot + tell. A fact for the caller choosing ``allowed_sources``; optional -- + a driver that cannot tell does not implement it.""" + ... + + # The steps a driver implements: + + def _install_snmp_agent(self) -> FixStepOutcomeDict: ... + + def _write_snmp_agent_config( + self, settings: SnmpAgentSettingsDict + ) -> FixStepOutcomeDict: ... + + def _allow_snmp_sources(self, settings: SnmpAgentSettingsDict) -> FixStepOutcomeDict: + """Let ``settings["allowed_sources"]`` reach the agent; ``ok`` with nothing + to do where the host has no firewall of its own.""" + ... + + def _start_snmp_agent(self) -> FixStepOutcomeDict: + """(Re)start the agent so it reads the new configuration, and enable it at boot.""" + ... + + +class EnsurePackageMixin: + """Make sure a package is installed, installing it only where it is missing. + + Works over :class:`~napalm_device_types.packages.PackageManagementMixin`'s + ``get_packages`` and ``install_package``, which the driver implements. + """ + + def ensure_package(self, name: str) -> FixResultDict: + """Install *name* unless it is installed already.""" + + def install() -> FixStepOutcomeDict: + if any(p.get("name") == name and p.get("installed") for p in self.get_packages()): + return {"ok": True, "output": f"{name} is already installed"} + result: Any = self.install_package(name) # type: ignore[func-returns-value] + # The contract says None (and raise on failure); some drivers still + # return {"success", "output"}. + if isinstance(result, dict): + return {"ok": bool(result.get("success")), "output": str(result.get("output", ""))} + return {"ok": True, "output": f"installed {name}"} + + return run_fix_steps([("install", install)]) + + if TYPE_CHECKING: # pragma: no cover + + def get_packages(self) -> list[PackageDict]: ... + + def install_package(self, name: str, version: str = "") -> None: ... + + +class LldpAgentMixin: + """Run an LLDP agent on the device, on the interfaces the caller names. + + The steps: install the agent, write the interfaces it runs on, enable it at boot + and restart it so it reads them. Installing and starting are generic, over + ``ensure_package`` and ``manage_service`` with ``LLDP_AGENT_PACKAGE`` and + ``LLDP_AGENT_SERVICE``; the driver writes the configuration. Which interfaces is + the caller's choice -- an agent on every interface also talks to the uplink of a + WAN port. Whether neighbours appear is the caller's to read with + ``get_lldp_neighbors``, after the agent's first advertisement interval. + """ + + #: The package that brings the agent, as ``ensure_package`` takes it. + LLDP_AGENT_PACKAGE: ClassVar[str] = "lldpd" + #: The agent's service, as ``manage_service`` takes it. + LLDP_AGENT_SERVICE: ClassVar[str] = "lldpd" + + def configure_lldp_agent(self, interfaces: Sequence[str]) -> FixResultDict: + """Install, configure for *interfaces*, enable and restart the agent. + + :raises ValueError: for no interface, before anything changes. + """ + names = [str(name) for name in interfaces] + if not names: + raise ValueError("name at least one interface for the LLDP agent") + return run_fix_steps( + [ + ("install", self._install_lldp_agent), + ("configure", lambda: self._write_lldp_agent_config(names)), + ("start", self._start_lldp_agent), + ] + ) + + def _install_lldp_agent(self) -> FixStepOutcomeDict: + result = self.ensure_package(self.LLDP_AGENT_PACKAGE) + return {"ok": bool(result["success"]), "output": str(result.get("output", ""))} + + def _start_lldp_agent(self) -> FixStepOutcomeDict: + service = self.LLDP_AGENT_SERVICE + for action in ("enable", "restart"): + result = self.manage_service(service, action) + if not result.get("success"): + output = str(result.get("output", "")).strip() + return {"ok": False, "output": f"{action} {service}: {output}".rstrip(": ")} + return {"ok": True, "output": f"{service} enabled and restarted"} + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def ensure_package(self, name: str) -> FixResultDict: ... + + def manage_service(self, name: str, action: str) -> dict[str, Any]: ... + + # The step a driver implements: + + def _write_lldp_agent_config(self, interfaces: list[str]) -> FixStepOutcomeDict: + """Have the agent send and listen on exactly *interfaces* (names as the + device's interface getters report them), replacing the list it had.""" + ... + + +class PackageProxyMixin: + """Point the device's package manager at a caching proxy.""" + + if TYPE_CHECKING: # pragma: no cover + + def configure_package_proxy(self, url: str) -> FixResultDict: + """Have the package manager fetch through *url*; an empty *url* removes it.""" + ... + + def get_package_proxy(self) -> str | None: + """The proxy the package manager fetches through now; None for none. + + A fact: whether it is the right one is the caller's to judge.""" + ... + + +__all__ = [ + "EnsurePackageMixin", + "FixStep", + "LldpAgentMixin", + "PackageProxyMixin", + "SnmpAgentMixin", + "run_fix_steps", +] diff --git a/napalm_device_types/health_metrics.py b/napalm_device_types/health_metrics.py index 631251d..de0bd45 100644 --- a/napalm_device_types/health_metrics.py +++ b/napalm_device_types/health_metrics.py @@ -1,6 +1,10 @@ -# -*- coding: utf-8 -*- """SNMP health collection, shared by every device type that speaks UCD-MIB. +**Deprecated** (docs/STRUCTURE.md): drivers do not collect over SNMP. The +application polls; a driver whose device publishes values under its own OIDs +declares them as ``SNMP_HEALTH_PROFILE`` +(:class:`~napalm_device_types.snmp_health_profile.SnmpHealthProfile`). This mixin goes in 5.0. + The collection itself is generic: CPU, memory, uptime, load and per-interface counters come from the same standard OIDs on any UCD-MIB/IF-MIB capable device. Only two details vary by device type, and both are class attributes rather than @@ -15,7 +19,9 @@ implement ``get_health_metrics`` themselves. from __future__ import annotations import re -from typing import Any, Awaitable, Callable, ClassVar, Dict, cast +import warnings +from collections.abc import Awaitable, Callable +from typing import Any, ClassVar, cast from napalm_device_types._ucd_metrics import IF_SKIP_DEFAULT, collect_ucd_metrics from napalm_device_types.models import HealthMetricsDict @@ -37,9 +43,19 @@ class HealthMetricsMixin: async def get_health_metrics( cls, snmp_get: Callable[..., Awaitable[Any]], - snmp_walk: Callable[..., Awaitable[Dict[str, Any]]], + snmp_walk: Callable[..., Awaitable[dict[str, Any]]], ) -> HealthMetricsDict: - """Collect SNMP health metrics (CPU, memory, uptime, load, interfaces).""" + """Collect SNMP health metrics (CPU, memory, uptime, load, interfaces). + + Deprecated: the application polls; see the module docstring. + """ + warnings.warn( + "HealthMetricsMixin.get_health_metrics is deprecated and goes in 5.0: " + "the application polls over SNMP; a driver declares only its OIDs, " + "as SNMP_HEALTH_PROFILE", + DeprecationWarning, + stacklevel=2, + ) # collect_ucd_metrics is untyped shared plumbing; the shape it returns is # the contract HealthMetricsDict describes. return cast( diff --git a/napalm_device_types/host_firewall.py b/napalm_device_types/host_firewall.py new file mode 100644 index 0000000..3b03deb --- /dev/null +++ b/napalm_device_types/host_firewall.py @@ -0,0 +1,770 @@ +"""A Linux host's own firewall: its netfilter rules, read once, in one shape. + +Whether a service is reachable from outside its host is decided twice: by the +address it listens on (:mod:`napalm_device_types.listening`), and by the host's +own firewall, which may drop the packets first. Reading the firewall is the +same on every Linux host, so the command and its parse live here, and a driver +only carries the command across. + +**What is read.** ``nft -j list ruleset`` is the primary reading: every table, +the host's own ``nftables.service`` ones included. Rules that iptables-nft wrote +are in it too, but their extensions show only as an opaque ``xt`` -- a +``-m conntrack --ctstate`` or ``-m set`` says nothing there. ``iptables-save`` +spells them out, so the tables iptables manages are taken from it, and the +others from nft. A *legacy* iptables is a second ruleset beside nft, and both +apply to every packet, so then both are kept whole. + +**One shape.** Each chain says its family, table, hook, priority and policy; +each rule a list of matches and a verdict. A match is ``field``, ``op`` +(``==``/``!=``) and ``values``. The fields are the few that decide reachability +from outside: ``iifname``, ``oifname``, ``saddr``, ``daddr``, ``l4proto``, +``dport``, ``sport``, ``ct_state``, ``ct_orig_dport`` and ``nfproto``. +Everything else -- an ipset, ``addrtype``, a rate limit, a ``fib`` lookup -- is +``unknown`` with its text, never dropped: a rule that matches on something this +does not model must not read as one that matches everything. + +**Root, and only root.** Both tools need ``CAP_NET_ADMIN``; without it they +print an error, not an empty ruleset. So there is no reading without root: +:class:`HostFirewallUnavailable` is raised rather than a host without rules. + +Whether a rule set makes a service unreachable is the consumer's call. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +import shlex +from collections.abc import Iterator +from shlex import quote +from typing import TYPE_CHECKING, Any + +from napalm_device_types.models import ( + HostFirewallDict, + NetfilterChainDict, + NetfilterMatchDict, + NetfilterRuleDict, +) +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "FW_BEGIN" +_END = "FW_END" +_RC_RE = re.compile(r"^__FW_RC=(\d+)$") + +#: One line, POSIX ``sh``, read-only. The frame markers are printed in two +#: halves so that a transport which echoes the command does not show them early. +#: Both tools live in ``/usr/sbin`` on most systems, outside a login user's ``PATH``. +HOST_FIREWALL_COMMAND = ( + "PATH=$PATH:/usr/sbin:/sbin; " + "printf '%s%s\\n' FW_ BEGIN; " + "if command -v nft >/dev/null 2>&1; then " + "echo '[nft]'; nft -j list ruleset 2>&1; echo \"__FW_RC=$?\"; fi; " + "for t in iptables-save ip6tables-save; do " + "if command -v $t >/dev/null 2>&1; then " + 'echo "[$t]"; $t 2>&1; echo "__FW_RC=$?"; fi; done; ' + "printf '%s%s\\n' FW_ END" +) + +#: ``iptables-save`` reads the IPv4 tables, ``ip6tables-save`` the IPv6 ones. +_IPTABLES_FAMILY = {"iptables-save": "ip", "ip6tables-save": "ip6"} + +#: The built-in chains of iptables' tables: ``(type, hook, priority)``, as +#: iptables-nft registers them (and as the legacy tables run in the kernel). +_BUILT_IN: dict[str, dict[str, tuple[str, str, int]]] = { + "raw": {"PREROUTING": ("filter", "prerouting", -300), "OUTPUT": ("filter", "output", -300)}, + "mangle": { + "PREROUTING": ("filter", "prerouting", -150), + "INPUT": ("filter", "input", -150), + "FORWARD": ("filter", "forward", -150), + "OUTPUT": ("route", "output", -150), + "POSTROUTING": ("filter", "postrouting", -150), + }, + "nat": { + "PREROUTING": ("nat", "prerouting", -100), + "INPUT": ("nat", "input", 100), + "OUTPUT": ("nat", "output", -100), + "POSTROUTING": ("nat", "postrouting", 100), + }, + "filter": { + "INPUT": ("filter", "input", 0), + "FORWARD": ("filter", "forward", 0), + "OUTPUT": ("filter", "output", 0), + }, + "security": { + "INPUT": ("filter", "input", 50), + "FORWARD": ("filter", "forward", 50), + "OUTPUT": ("filter", "output", 50), + }, +} + +_TERMINAL = {"ACCEPT": "accept", "DROP": "drop", "REJECT": "reject", "RETURN": "return"} +#: Targets that look at a packet, mark it or log it, and let it go on. +_NON_TERMINAL = frozenset( + { + "LOG", + "NFLOG", + "ULOG", + "TRACE", + "MARK", + "CONNMARK", + "CT", + "NOTRACK", + "TCPMSS", + "CHECKSUM", + "AUDIT", + "SET", + "DSCP", + "TOS", + "TTL", + "HL", + "CLASSIFY", + "CONNSECMARK", + "SECMARK", + "IDLETIMER", + "LED", + "RATEEST", + } +) + + +class HostFirewallUnavailable(NotImplementedError): + """Nothing on the host could be read: no ``nft``, no ``iptables-save``, or no root.""" + + +def _frame(output: str) -> list[str]: + lines = [line.rstrip() for line in strip_terminal_codes(output).splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact host firewall report in the output") from None + return lines[start + 1 : end] + + +def _sections(lines: list[str]) -> dict[str, tuple[list[str], int | None]]: + """Each tool's output and exit status.""" + sections: dict[str, tuple[list[str], int | None]] = {} + name: str | None = None + body: list[str] = [] + for line in lines: + text = line.strip() + if text in ("[nft]", "[iptables-save]", "[ip6tables-save]"): + name, body = text[1:-1], [] + sections[name] = (body, None) + continue + rc = _RC_RE.match(text) + if rc and name is not None: + sections[name] = (body, int(rc.group(1))) + name = None + continue + if name is not None: + body.append(line) + return sections + + +# ── iptables-save ──────────────────────────────────────────────────────────── + +#: The token that negates the option after it (``! -s 10.0.0.0/8``). +_NEGATION = "!" + +#: Options that start a new part of a rule, outside any ``-m`` module. +_TOP_LEVEL = frozenset( + { + "-m", + "--match", + "-j", + "--jump", + "-g", + "--goto", + "-p", + "--protocol", + "-s", + "--source", + "-d", + "--destination", + "-i", + "--in-interface", + "-o", + "--out-interface", + "-f", + "--fragment", + "-c", + "--set-counters", + } +) +#: Modules whose options this reads; any other module is an ``unknown`` match. +_KNOWN_MODULES = frozenset( + {"tcp", "udp", "sctp", "udplite", "multiport", "state", "conntrack", "comment"} +) + + +def _match(field: str, negated: bool, values: list[str]) -> NetfilterMatchDict: + return {"field": field, "op": "!=" if negated else "==", "values": values} + + +def _unknown(text: str) -> NetfilterMatchDict: + return {"field": "unknown", "op": "==", "values": [text]} + + +def _iface(name: str) -> str: + """iptables writes a wildcard interface as ``eth+``; nft as ``eth*``.""" + return name[:-1] + "*" if name.endswith("+") else name + + +def _ports(spec: str) -> list[str]: + return [p.replace(":", "-") for p in spec.split(",") if p] + + +def _states(spec: str) -> list[str]: + return [s.lower() for s in spec.split(",") if s] + + +class _RuleTokens: + """The tokens of one ``-A`` line, read front to back.""" + + def __init__(self, tokens: list[str]): + self.tokens = tokens + self.i = 0 + + def more(self) -> bool: + return self.i < len(self.tokens) + + def peek(self, ahead: int = 0) -> str | None: + j = self.i + ahead + return self.tokens[j] if j < len(self.tokens) else None + + def take(self) -> str: + token = self.tokens[self.i] + self.i += 1 + return token + + +def _module_options(module: str, rt: _RuleTokens, matches: list[NetfilterMatchDict]) -> None: + """Read *module*'s options from *rt* into *matches*; stop at the next top-level option.""" + text = ["-m", module] + unknown = module not in _KNOWN_MODULES + while rt.more(): + token = rt.peek() + if token in _TOP_LEVEL or (token == _NEGATION and rt.peek(1) in _TOP_LEVEL): + break + negated = False + if token == _NEGATION: + rt.take() + negated = True + text.append(_NEGATION) + option = rt.take() + value = rt.take() if rt.more() and not str(rt.peek()).startswith("-") else "" + text += [option] + ([value] if value else []) + if unknown: + continue + mapped = _module_match(module, option, value, negated) + if mapped is None: + unknown = True + elif mapped is not False: + matches.append(mapped) + if unknown and module != "comment": + matches.append(_unknown(" ".join(text))) + + +def _module_match(module: str, option: str, value: str, negated: bool) -> Any: + """The match one module option makes. + + False for one that matches nothing new; None if not read. + """ + if module == "comment": + return False + if module in ("tcp", "udp", "sctp", "udplite"): + if option in ("--dport", "--destination-port"): + return _match("dport", negated, _ports(value)) + if option in ("--sport", "--source-port"): + return _match("sport", negated, _ports(value)) + return None + if module == "multiport": + if option in ("--dports", "--destination-ports"): + return _match("dport", negated, _ports(value)) + if option in ("--sports", "--source-ports"): + return _match("sport", negated, _ports(value)) + return None + if module == "state" and option == "--state": + return _match("ct_state", negated, _states(value)) + if module == "conntrack": + if option == "--ctstate": + return _match("ct_state", negated, _states(value)) + if option == "--ctorigdstport": + return _match("ct_orig_dport", negated, _ports(value)) + return None + + +def _target(name: str, chains: set[str]) -> tuple[str, str | None]: + if name in _TERMINAL: + return _TERMINAL[name], None + if name in chains: + return "jump", name + if name in _NON_TERMINAL: + return "continue", name + return "other", name + + +def _iptables_rule(line: str, chains: set[str]) -> tuple[str, NetfilterRuleDict]: + """``-A CHAIN ...`` as ``(chain, rule)``.""" + tokens = shlex.split(line) + chain = tokens[1] + rt = _RuleTokens(tokens[2:]) + matches: list[NetfilterMatchDict] = [] + verdict, target = "continue", None + negated = False + simple = { + "-i": "iifname", + "--in-interface": "iifname", + "-o": "oifname", + "--out-interface": "oifname", + "-s": "saddr", + "--source": "saddr", + "-d": "daddr", + "--destination": "daddr", + "-p": "l4proto", + "--protocol": "l4proto", + } + while rt.more(): + token = rt.take() + if token == _NEGATION: + negated = True + continue + if token in simple: + field, value = simple[token], rt.take() + if field in ("iifname", "oifname"): + matches.append(_match(field, negated, [_iface(value)])) + elif field == "l4proto": + if value.lower() != "all": + matches.append(_match(field, negated, [value.lower()])) + else: + matches.append(_match(field, negated, value.split(","))) + elif token in ("-m", "--match"): + _module_options(rt.take(), rt, matches) + elif token in ("-j", "--jump"): + verdict, target = _target(rt.take(), chains) + break # the target's own options follow and match nothing + elif token in ("-g", "--goto"): + verdict, target = "goto", rt.take() + break + elif token in ("-c", "--set-counters"): + rt.take(), rt.take() + else: + matches.append(_unknown(token)) + negated = False + return chain, {"matches": matches, "verdict": verdict, "target": target, "text": line} + + +def _iptables_variant(lines: list[str]) -> str | None: + """``iptables-nft`` or ``iptables-legacy`` from the header; None if nothing was saved.""" + for line in lines: + if line.startswith("# Generated by"): + return "iptables-nft" if "(nf_tables)" in line else "iptables-legacy" + return None + + +def _parse_iptables_save(lines: list[str], family: str) -> list[NetfilterChainDict]: + chains: list[NetfilterChainDict] = [] + table: str | None = None + by_name: dict[str, NetfilterChainDict] = {} + for raw in lines: + line = raw.strip() + if line.startswith("*"): + table, by_name = line[1:], {} + elif line.startswith(":") and table is not None: + name, policy = line[1:].split()[:2] + kind, hook, prio = _BUILT_IN.get(table, {}).get(name, (None, None, None)) + chain: NetfilterChainDict = { + "family": family, + "table": table, + "name": name, + "type": kind, + "hook": hook, + "priority": prio, + "policy": None if policy == "-" else policy.lower(), + "rules": [], + } + by_name[name] = chain + chains.append(chain) + elif line.startswith("-A ") and table is not None: + name, rule = _iptables_rule(line, set(by_name)) + if name in by_name: + by_name[name]["rules"].append(rule) + return chains + + +# ── nft -j ─────────────────────────────────────────────────────────────────── + +_VERDICTS = { + "accept": "accept", + "drop": "drop", + "reject": "reject", + "return": "return", + "continue": "continue", +} +#: Statements that act on a packet without deciding its fate. +_NO_OPS = frozenset( + { + "counter", + "log", + "mangle", + "notrack", + "set", + "flow", + "ct helper", + "synproxy", + "tcp option", + "dup", + "fwd", + } +) +#: Statements that pass only some packets on: what follows is not about every packet. +_PARTIAL = frozenset({"limit", "quota", "numgen", "hash", "ct count"}) +#: Verdicts that hand the packet elsewhere, or that this does not model. +_OTHER = frozenset({"queue", "dnat", "snat", "masquerade", "redirect", "tproxy", "vmap"}) +_TRANSPORTS = frozenset({"tcp", "udp", "sctp", "udplite", "dccp"}) + + +def _nft_scalar(value: Any) -> list[str] | None: + """A right-hand side as values; None for one this does not model.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, str)): + return [str(value)] + if isinstance(value, dict): + if "prefix" in value: + return [f"{value['prefix']['addr']}/{value['prefix']['len']}"] + if "range" in value: + low, high = value["range"] + return [f"{low}-{high}"] + if "elem" in value: + return _nft_scalar(value["elem"].get("val")) + if "set" in value: + return _nft_values(value["set"]) + if isinstance(value, list): + return _nft_values(value) + return None + + +def _nft_values(items: Any) -> list[str] | None: + items = items if isinstance(items, list) else [items] + out: list[str] = [] + for item in items: + values = _nft_scalar(item) + if values is None: + return None + out += values + return out + + +def _left_text(left: Any) -> str: + if isinstance(left, dict): + if "payload" in left: + return ( + f"{left['payload'].get('protocol', '')} {left['payload'].get('field', '')}".strip() + ) + if "meta" in left: + return f"meta {left['meta'].get('key', '')}" + if "ct" in left: + ct = left["ct"] + return f"ct {ct.get('dir', '')} {ct.get('key', '')}".replace(" ", " ") + key = next(iter(left), "") + return str(key) + return str(left) + + +def _right_text(right: Any) -> str: + values = _nft_scalar(right) + if values is None: + return json.dumps(right, sort_keys=True) + return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" + + +def _nft_field(left: Any) -> tuple[str | None, str | None]: + """``(field, implied l4proto)`` of a left-hand side; ``(None, None)`` if not modelled.""" + if not isinstance(left, dict): + return None, None + if "meta" in left: + key = left["meta"].get("key") + return { + "iifname": "iifname", + "iif": "iifname", + "oifname": "oifname", + "oif": "oifname", + "l4proto": "l4proto", + "nfproto": "nfproto", + }.get(key), None + if "payload" in left: + proto, field = left["payload"].get("protocol"), left["payload"].get("field") + if proto in ("ip", "ip6") and field in ("saddr", "daddr"): + return field, None + if (proto, field) in (("ip", "protocol"), ("ip6", "nexthdr")): + return "l4proto", None + if field in ("dport", "sport") and (proto in _TRANSPORTS or proto == "th"): + return field, proto if proto in _TRANSPORTS else None + return None, None + if "ct" in left: + key, direction = left["ct"].get("key"), left["ct"].get("dir") + if key == "state": + return "ct_state", None + if key == "proto-dst" and direction == "original": + return "ct_orig_dport", None + return None, None + + +def _nft_match( + match: dict[str, Any], sets: dict[tuple[str, str, str], list[str]], where: tuple[str, str] +) -> list[NetfilterMatchDict]: + left, op, right = match.get("left"), match.get("op"), match.get("right") + field, implied = _nft_field(left) + raw_text = ( + f"{_left_text(left)} {'' if op in ('==', 'in') else str(op) + ' '}{_right_text(right)}" + ) + if field is None or op not in ("==", "in", "!="): + return [_unknown(raw_text)] + if isinstance(right, str) and right.startswith("@"): + values = sets.get((where[0], where[1], right[1:])) + else: + values = _nft_scalar(right) + if values is None: + return [_unknown(raw_text)] + out: list[NetfilterMatchDict] = [] + if implied: + out.append(_match("l4proto", False, [implied])) + if field in ("iifname", "oifname", "l4proto", "ct_state", "nfproto"): + values = [ + str(v).lower() if field != "iifname" and field != "oifname" else str(v) for v in values + ] + out.append(_match(field, op == "!=", values)) + return out + + +def _nft_rule( + rule: dict[str, Any], sets: dict[tuple[str, str, str], list[str]] +) -> NetfilterRuleDict: + where = (rule.get("family", ""), rule.get("table", "")) + matches: list[NetfilterMatchDict] = [] + verdict, target = "continue", None + for statement in rule.get("expr") or []: + if not isinstance(statement, dict) or not statement: + continue + key = next(iter(statement)) + body = statement[key] + if key == "match": + matches += _nft_match(body, sets, where) + elif key in _VERDICTS: + verdict = _VERDICTS[key] + break + elif key in ("jump", "goto"): + verdict, target = key, str((body or {}).get("target")) + break + elif key == "xt": + if (body or {}).get("type") == "target": + verdict, target = "other", str(body.get("name")) + break + matches.append(_unknown(f"xt match {(body or {}).get('name')}")) + elif key in _PARTIAL: + matches.append(_unknown(key)) + elif key in _OTHER: + verdict, target = "other", key + break + elif key not in _NO_OPS: + matches.append(_unknown(key)) + entry: NetfilterRuleDict = { + "matches": matches, + "verdict": verdict, + "target": target, + "text": "", + } + entry["text"] = rule_text(entry) + return entry + + +def _parse_nft(text: str, skip: set[tuple[str, str]]) -> list[NetfilterChainDict]: + start = text.find("{") + if start < 0: + raise ValueError("nft printed no JSON") + objects = json.loads(text[start:]).get("nftables") or [] + sets: dict[tuple[str, str, str], list[str]] = {} + for obj in objects: + if "set" in obj and "elem" in obj["set"]: + values = _nft_values(obj["set"]["elem"]) + if values is not None: + s = obj["set"] + sets[(s.get("family", ""), s.get("table", ""), s.get("name", ""))] = values + chains: dict[tuple[str, str, str], NetfilterChainDict] = {} + for obj in objects: + if "chain" not in obj: + continue + c = obj["chain"] + if (c.get("family"), c.get("table")) in skip: + continue + chains[(c["family"], c["table"], c["name"])] = { + "family": c["family"], + "table": c["table"], + "name": c["name"], + "type": c.get("type"), + "hook": c.get("hook"), + "priority": c.get("prio"), + "policy": c.get("policy") if c.get("hook") else None, + "rules": [], + } + for obj in objects: + if "rule" not in obj: + continue + r = obj["rule"] + chain = chains.get((r.get("family"), r.get("table"), r.get("chain"))) + if chain is not None: + chain["rules"].append(_nft_rule(r, sets)) + return list(chains.values()) + + +# ── text ───────────────────────────────────────────────────────────────────── + + +def _family_of(values: list[str]) -> str: + try: + return ( + "ip6" + if all(ipaddress.ip_network(v, strict=False).version == 6 for v in values) + else "ip" + ) + except ValueError: + return "ip" + + +def _values_text(values: list[str]) -> str: + return values[0] if len(values) == 1 else "{" + ", ".join(values) + "}" + + +def rule_text(rule: NetfilterRuleDict) -> str: + """A rule as one nft-style line, from its normalized matches and verdict.""" + protos = [ + v + for m in rule["matches"] + if m["field"] == "l4proto" and m["op"] == "==" + for v in m["values"] + ] + parts: list[str] = [] + for m in rule["matches"]: + neg = "!= " if m["op"] == "!=" else "" + field, values = m["field"], m["values"] + if field == "unknown": + parts.append(" ".join(values)) + continue + keyword = { + "saddr": f"{_family_of(values)} saddr", + "daddr": f"{_family_of(values)} daddr", + "l4proto": "meta l4proto", + "nfproto": "meta nfproto", + "ct_state": "ct state", + "ct_orig_dport": "ct original proto-dst", + "dport": f"{protos[0] if len(protos) == 1 else 'th'} dport", + "sport": f"{protos[0] if len(protos) == 1 else 'th'} sport", + }.get(field, field) + parts.append(f"{keyword} {neg}{_values_text(values)}") + verdict = rule["verdict"] + if verdict in ("jump", "goto"): + parts.append(f"{verdict} {rule['target']}") + elif verdict == "other": + parts.append(str(rule["target"] or "other")) + elif verdict != "continue": + parts.append(verdict) + return " ".join(parts) + + +# ── the reading ────────────────────────────────────────────────────────────── + + +def _iptables_sections( + sections: dict[str, tuple[list[str], int | None]], +) -> Iterator[tuple[str, str, list[str]]]: + """``(variant, family, lines)`` for each iptables tool that saved something.""" + for tool, family in _IPTABLES_FAMILY.items(): + lines, rc = sections.get(tool, ([], None)) + if rc != 0: + continue + variant = _iptables_variant(lines) + if variant is not None: + yield variant, family, lines + + +def parse_host_firewall(output: str) -> HostFirewallDict: + """The framed report of :data:`HOST_FIREWALL_COMMAND` as one ruleset. + + :raises ValueError: if the report's frame is not intact. + :raises HostFirewallUnavailable: if neither tool could be read. + """ + sections = _sections(_frame(output)) + sources: list[str] = [] + chains: list[NetfilterChainDict] = [] + covered: set[tuple[str, str]] = set() + for variant, family, lines in _iptables_sections(sections): + saved = _parse_iptables_save(lines, family) + chains += saved + if variant not in sources: + sources.append(variant) + if variant == "iptables-nft": + covered |= {(c["family"], c["table"]) for c in saved} + nft_lines, nft_rc = sections.get("nft", ([], None)) + if nft_rc == 0: + try: + nft_chains = _parse_nft("\n".join(nft_lines), covered) + except ValueError: + nft_chains = None + if nft_chains is not None: + sources.insert(0, "nftables") + chains = nft_chains + chains + if not sources: + raise HostFirewallUnavailable("neither nft nor iptables-save could be read (root needed)") + return {"sources": sources, "chains": chains} + + +class HostFirewallMixin: + """Adds :meth:`get_host_firewall` to a driver that can run a command on a Linux host. + + The template form (README, "Function classes"): the command and its parse + are the same everywhere, so they are concrete here, and a driver supplies + only :meth:`_run_host_firewall_command` -- how a command reaches its host, + and how it gains root there. Mixed in by the drivers that can, so + ``hasattr(driver, "get_host_firewall")`` stays truthful. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_host_firewall_command(self, command: str, *, privileged: bool) -> str: + """Run *command* on the host and return what it printed; as root + when *privileged*. The command is a single ``sh -c`` invocation.""" + ... + + def get_host_firewall(self) -> HostFirewallDict: + """ + Returns the host's netfilter rules: every chain with its hook, priority + and policy, and every rule as matches and a verdict. + + * sources (list) - ``nftables``, ``iptables-nft``, ``iptables-legacy``: what was read + * chains (list) - see :class:`~napalm_device_types.models.NetfilterChainDict` + + Example:: + + { + "sources": ["nftables", "iptables-nft"], + "chains": [ + {"family": "ip", "table": "filter", "name": "INPUT", "type": "filter", + "hook": "input", "priority": 0, "policy": "accept", + "rules": [ + {"matches": [{"field": "iifname", "op": "==", "values": ["ens6"]}], + "verdict": "drop", "target": None, + "text": "-A INPUT -i ens6 -j DROP"}, + ]}, + ], + } + + :raises HostFirewallUnavailable: if neither tool could be read -- no + ``nft``, no ``iptables-save``, or no root to run them. + :raises ValueError: if the report came back cut short. + """ + command = f"sh -c {quote(HOST_FIREWALL_COMMAND)}" + return parse_host_firewall(self._run_host_firewall_command(command, privileged=True)) diff --git a/napalm_device_types/host_shutdown.py b/napalm_device_types/host_shutdown.py new file mode 100644 index 0000000..e1e030b --- /dev/null +++ b/napalm_device_types/host_shutdown.py @@ -0,0 +1,32 @@ +"""Shutting the device itself down cleanly. + +Declared under ``if TYPE_CHECKING``: a contract, not a placeholder. Only a +driver that can shut its device down through the device's own means defines +``shutdown_host``, so ``hasattr(driver, "shutdown_host")`` tells a caller +whether to offer it. There is no forced variant here, by design: cutting the +power is the BMC's business, not a driver's. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + + +class HostShutdownMixin: + if TYPE_CHECKING: + + def shutdown_host(self) -> None: + """ + Shuts down the device this driver is connected to, cleanly. + + Returns once the device has accepted the request; the session is + usually gone right after. Whether the device really went off is + the caller's business, e.g. through its BMC. + + A driver that manages other machines shuts down its *own* host, + never one of them: a hypervisor node shuts down itself, and its + guests are stopped the way the node stops them on shutdown. + + :raises RuntimeError: If the device refuses the request. + """ + ... diff --git a/napalm_device_types/host_status.py b/napalm_device_types/host_status.py index ea257fa..68307fe 100644 --- a/napalm_device_types/host_status.py +++ b/napalm_device_types/host_status.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Host status: does the host need a reboot, and does it patch itself? A patch run that installed a new kernel or libc has not closed anything until @@ -59,7 +58,7 @@ too full is not decided here. from __future__ import annotations import re -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import ( FilesystemUsageDict, @@ -110,7 +109,7 @@ HOST_STATUS_COMMAND = ( "apt-config dump 2>/dev/null | grep '^APT::Periodic::Unattended-Upgrade '; " # Named through a variable: spelt out, the program name would match BUSY_CHECK # in this very command line. - "uu=unattended-upgrade; { command -v \"$uu\" || [ -x \"/usr/bin/$uu\" ]; } >/dev/null 2>&1 " + 'uu=unattended-upgrade; { command -v "$uu" || [ -x "/usr/bin/$uu" ]; } >/dev/null 2>&1 ' "&& echo '[unattended-upgrade]'; " f"echo '[upgrade-stamp]'; stat -c %Y {_UPGRADE_STAMP} 2>/dev/null; fi; " "if command -v dpkg-query >/dev/null 2>&1; then " @@ -147,15 +146,15 @@ _HALFWAY = { } -def _sections(output: str) -> Dict[str, List[str]]: +def _sections(output: str) -> dict[str, list[str]]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) end = lines.index(_END, start) except ValueError: raise ValueError("no intact host status report in the output") from None - sections: Dict[str, List[str]] = {} - current: List[str] = [] + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines[start + 1 : end]: if line.startswith("[") and line.endswith("]"): current = sections.setdefault(line[1:-1], []) @@ -164,12 +163,12 @@ def _sections(output: str) -> Dict[str, List[str]]: return sections -def _version_key(version: str) -> Tuple[object, ...]: +def _version_key(version: str) -> tuple[object, ...]: """Natural order: 6.8.0-142 after 6.8.0-87, 7.0.14 after 7.0.2.""" return tuple(int(part) if part.isdigit() else part for part in re.split(r"(\d+)", version)) -def kernel_reboot_pending(running: str, installed: List[str]) -> Optional[str]: +def kernel_reboot_pending(running: str, installed: list[str]) -> str | None: """The newest installed kernel of the running flavour, if it is newer than the running one; otherwise None. @@ -187,7 +186,7 @@ def kernel_reboot_pending(running: str, installed: List[str]) -> Optional[str]: return None -def _kernel_flavour(release: str) -> Tuple[str, str]: +def _kernel_flavour(release: str) -> tuple[str, str]: """``6.8.0-87-generic`` -> ``("6.8.0-87", "generic")``, ``5.14.0-427.13.1.el9_4.x86_64`` -> ``("5.14.0-427.13.1.el9_4", "x86_64")``.""" base, sep, tail = release.rpartition("-") @@ -199,7 +198,7 @@ def _kernel_flavour(release: str) -> Tuple[str, str]: return base, tail -def _freebsd_reboot(sections: Dict[str, List[str]]) -> Optional[Tuple[bool, Optional[str]]]: +def _freebsd_reboot(sections: dict[str, list[str]]) -> tuple[bool, str | None] | None: """FreeBSD's answer from freebsd-version, or None on any other system.""" installed = (sections.get("freebsd-kernel") or [""])[0] running = (sections.get("freebsd-running") or [""])[0] @@ -210,7 +209,7 @@ def _freebsd_reboot(sections: Dict[str, List[str]]) -> Optional[Tuple[bool, Opti return False, None -def _reboot(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], Optional[str]]: +def _reboot(sections: dict[str, list[str]]) -> tuple[bool | None, str | None]: if "reboot-required" in sections: return True, f"{_REBOOT_FILE} is present" needs = sections.get("needs-restarting") @@ -229,29 +228,29 @@ def _reboot(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], Optional[st return None, None -def _timer_states(sections: Dict[str, List[str]]) -> Dict[str, str]: - states: Dict[str, str] = {} +def _timer_states(sections: dict[str, list[str]]) -> dict[str, str]: + states: dict[str, str] = {} for line in sections.get("timers") or []: unit, _, state = line.partition(" ") states[unit] = state.strip() return states -def _periodic(sections: Dict[str, List[str]]) -> Optional[str]: +def _periodic(sections: dict[str, list[str]]) -> str | None: """The value of ``APT::Periodic::Unattended-Upgrade``, or None when it is not set.""" lines = sections.get("apt-config") or [] match = next(filter(None, (_PERIODIC.match(line) for line in lines)), None) return match.group(1) if match else None -def _dnf_applies(sections: Dict[str, List[str]]) -> bool: +def _dnf_applies(sections: dict[str, list[str]]) -> bool: """Whether dnf-automatic.timer installs what it downloads; dnf's default is no.""" lines = sections.get("dnf-automatic") or [] values = [m.group(1) for m in map(_APPLY_UPDATES.match, lines) if m] return bool(values) and values[-1].lower() in _TRUE -def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]: +def _auto_updates(sections: dict[str, list[str]]) -> bool | None: timers = _timer_states(sections) if timers.get(_DNF_INSTALL_TIMER) == "enabled": return True @@ -265,7 +264,7 @@ def _auto_updates(sections: Dict[str, List[str]]) -> Optional[bool]: return switched_on and installed and timers.get(_APT_TIMER) not in _OFF_STATES -def _interval(sections: Dict[str, List[str]]) -> Optional[int]: +def _interval(sections: dict[str, list[str]]) -> int | None: """apt's interval in seconds, the way ``apt.systemd.daily`` reads it; 0 for "always".""" value = _periodic(sections) if value == "always": @@ -276,19 +275,19 @@ def _interval(sections: Dict[str, List[str]]) -> Optional[int]: return int(match.group(1)) * _UNIT_SECONDS[match.group(2)] -def _last_success(sections: Dict[str, List[str]]) -> Optional[int]: +def _last_success(sections: dict[str, list[str]]) -> int | None: stamp = (sections.get("upgrade-stamp") or [""])[0] return int(stamp) if stamp.isdigit() else None -def _halfway(line: str) -> Optional[InterruptedPackageDict]: +def _halfway(line: str) -> InterruptedPackageDict | None: status, _, name = line.partition(" ") name = name.strip() state = "reinstall-required" if status[2:3] == "R" else _HALFWAY.get(status[1:2]) return {"name": name, "state": state} if name and state else None -def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[InterruptedPackageDict]]: +def _dpkg(sections: dict[str, list[str]]) -> tuple[bool | None, list[InterruptedPackageDict]]: if "dpkg-audit" not in sections or "dpkg-busy" in sections: return None, [] packages = [p for p in map(_halfway, sections["dpkg-audit"]) if p is not None] @@ -296,9 +295,9 @@ def _dpkg(sections: Dict[str, List[str]]) -> Tuple[Optional[bool], List[Interrup return bool(packages) or journal, packages -def _filesystems(sections: Dict[str, List[str]]) -> Optional[List[FilesystemUsageDict]]: +def _filesystems(sections: dict[str, list[str]]) -> list[FilesystemUsageDict] | None: """Each filesystem df named, once and in its order; None when it named none.""" - found: Dict[str, FilesystemUsageDict] = {} + found: dict[str, FilesystemUsageDict] = {} for line in sections.get("filesystems") or []: match = _DF_LINE.match(line) if not match or int(match.group(1)) == 0: diff --git a/napalm_device_types/hypervisor.py b/napalm_device_types/hypervisor.py index 32173a4..8784884 100644 --- a/napalm_device_types/hypervisor.py +++ b/napalm_device_types/hypervisor.py @@ -10,24 +10,28 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.clusters import ClusterDict from napalm_device_types.health_metrics import HealthMetricsMixin -from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents from napalm_device_types.models import ( - NICConfigDict, + HypervisorHostDict, NetworkTargetDict, + NICConfigDict, SnapshotDict, StorageTargetDict, StorageVolumeDict, + VirtualNetworkDict, VMConfigDict, VMCpuTypeDict, VMDict, VMProvisionResultDict, VMStatusDict, - VirtualNetworkDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.provisioning import QEMU_GUEST_AGENTS, GuestAgents +from napalm_device_types.vm_migration import MigrationPreflightDict class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver): @@ -51,14 +55,13 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri #: the driver can provision; netOrk reads this off the class. GUEST_AGENTS: GuestAgents = {"linux": QEMU_GUEST_AGENTS["linux"]} - # ------------------------------------------------------------------ # Virtual machines – read # ------------------------------------------------------------------ if TYPE_CHECKING: - def get_vms(self) -> List[VMDict]: + def get_vms(self) -> list[VMDict]: """ Returns a list of all virtual machines known to this hypervisor, including their runtime status. @@ -261,7 +264,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Snapshots # ------------------------------------------------------------------ - def get_vm_snapshots(self, name: str) -> List[SnapshotDict]: + def get_vm_snapshots(self, name: str) -> list[SnapshotDict]: """ Returns all snapshots of a virtual machine. @@ -300,8 +303,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def create_vm_snapshot(self, name: str, snapshot: str, - description: str = "", include_memory: bool = False) -> None: + def create_vm_snapshot( + self, name: str, snapshot: str, description: str = "", include_memory: bool = False + ) -> None: """ Creates a snapshot of a virtual machine. @@ -328,7 +332,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri :param name: VM name or its ``vmid``. :param snapshot: Name of the snapshot to delete. :raises ValueError: If the VM or snapshot does not exist. - :raises RuntimeError: If other snapshots depend on this one (must delete children first). + :raises RuntimeError: If other snapshots depend on this one + (must delete children first). Example:: @@ -358,7 +363,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Storage # ------------------------------------------------------------------ - def get_vm_storage_pools(self) -> Dict[str, StorageVolumeDict]: + def get_vm_storage_pools(self) -> dict[str, StorageVolumeDict]: """ Returns the storage pools / datastores configured on this hypervisor. @@ -402,7 +407,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Virtual networking # ------------------------------------------------------------------ - def get_virtual_networks(self) -> Dict[str, VirtualNetworkDict]: + def get_virtual_networks(self) -> dict[str, VirtualNetworkDict]: """ Returns the virtual networks / bridges defined on this hypervisor. @@ -442,11 +447,6 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri # Package management (hypervisor extensions / plugins) # ------------------------------------------------------------------ - - - - - # ------------------------------------------------------------------ # Virtual machine provisioning # ------------------------------------------------------------------ @@ -458,29 +458,26 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri image_url: str, cpu: int, memory: int, - nics: List[NICConfigDict], - cloud_init_config: Dict[str, Any], + nics: list[NICConfigDict], + cloud_init_config: dict[str, Any], image_checksum: str | None = None, - ssh_public_keys: List[str] | None = None, + ssh_public_keys: list[str] | None = None, disk_resize_gb: int | None = None, storage: str | None = None, cpu_type: str | None = None, guest_os: str = "linux", + node: str | None = None, download_timeout: int = 300, timeout: int = 180, ) -> VMProvisionResultDict: """ Create a new virtual machine from a cloud image via Cloud-Init. - Downloads the cloud image (qcow2/raw) directly on the hypervisor if not - already cached there, creates a new VM shell, imports the image as its - root disk, configures virtual network interfaces, and injects Cloud-Init - configuration via a storage snippet or similar mechanism. The resulting - VM is left in a running state. - - Implementations should cache downloaded images by URL/filename on the - hypervisor so repeated provisioning from the same image does not - re-download it every time. + Downloads the cloud image (qcow2/raw) directly on the hypervisor unless it is + cached there already (implementations cache by URL/filename), creates a new VM + shell, imports the image as its root disk, configures virtual network interfaces, + and injects Cloud-Init configuration via a storage snippet or similar mechanism. + The resulting VM is left in a running state. Args: name (string) - new VM display name @@ -499,10 +496,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri image_checksum (string | None) - expected checksum of the downloaded image (e.g. "sha256:"). If given, verified after download; mismatch raises RuntimeError. If None, no verification is performed. - ssh_public_keys (list[str] | None) - SSH public keys to inject into guest. - If None or empty, no SSH key injection is performed. - disk_resize_gb (int | None) - resize root disk to this size in GB. - If None, disk remains the downloaded image's native size. Default None. + ssh_public_keys (list[str] | None) - SSH public keys to inject (None: none). + disk_resize_gb (int | None) - resize root disk to this size in GB (None: keep). storage (string | None) - name of the storage pool to place the root disk on (a name returned by ``get_image_storages()``). If None, the driver auto-detects the first enabled, node-available storage @@ -521,6 +516,9 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri network-config itself (``provisioning.network_config``). A guest_os the driver does not list raises ValueError before anything is created. Default ``"linux"``. + node (string | None) - a node ``name`` from ``get_clusters()`` to create + the VM on; None leaves it to the driver. One that only creates on its + own node raises ValueError for another, before creating anything. download_timeout (int) - maximum seconds to wait for the image download (skipped entirely if already cached on the hypervisor). Default 300. timeout (int) - maximum seconds to wait for the remaining provisioning @@ -533,6 +531,8 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri Raises: RuntimeError - if provisioning fails (download failure, checksum mismatch, storage unavailable, invalid config, timeout, etc.) + VMProvisionIncomplete - a RuntimeError for a failure after the VM was created, + with that VM's ``vmid`` and ``node`` for ``destroy_vm`` and the failure as cause. """ ... @@ -601,7 +601,7 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def get_network_targets(self) -> List[NetworkTargetDict]: + def get_network_targets(self) -> list[NetworkTargetDict]: """ List the network targets a new VM's NIC may attach to. @@ -615,11 +615,13 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri List[NetworkTargetDict] - each entry's ``vlan_aware`` flag tells the caller whether a ``NICConfigDict.vlan_tag`` may additionally be set for a NIC using that target (see ``NetworkTargetDict`` for the - per-kind rules). + per-kind rules). ``uplinks``, where the driver can tell, names + the physical NICs a bridge leaves the host through, grouped: + a bond's slaves in one group, a single NIC in its own. """ ... - def get_image_storages(self) -> List[StorageTargetDict]: + def get_image_storages(self) -> list[StorageTargetDict]: """ List the storage pools a new VM's root disk may be placed on, scoped to the specific node the VM will be created on. @@ -637,7 +639,71 @@ class HypervisorDriver(PackageManagementMixin, HealthMetricsMixin, DeviceTypeDri """ ... - def get_vm_cpu_types(self) -> List[VMCpuTypeDict]: + def get_clusters(self) -> list[ClusterDict]: + """ + List the clusters this connection can see, each with its nodes, its + storages and its networks. + + A cluster member (a Proxmox node) reports its one cluster, the whole of + it: every node, offline ones included with ``status`` ``"offline"``, and + ``local`` set on the node it is. A manager (vCenter) reports each + cluster it runs, and each standalone host it runs as a cluster of one. + A standalone host reports itself as a cluster of one, ``name`` None. + + The VMs ``get_vms()`` lists name their node in ``node``; that name is + the node's ``name`` here. A storage or network lists the nodes it is + available on, so a caller can tell where a VM may go. A value the + platform does not report is None, never a guess. + + Facts only: which node is a good place for a VM is the caller's to + decide. + + Returns: + List[ClusterDict] + """ + ... + + def get_vm_migration_preflight(self, name: str) -> MigrationPreflightDict: + """Which nodes of its cluster the guest could move to, and what blocks it + (see ``vm_migration``). + + :raises ValueError: If no VM with the given name/ID exists. + """ + ... + + def migrate_vm( + self, + name: str, + target_node: str, + online: bool = True, + target_storage: str | None = None, + timeout: int = 3600, + ) -> None: + """Move the guest to another node of its cluster; block until the + platform's task ended. ``online`` keeps a running guest running. + + :raises ValueError: Unknown VM, or a target that is not another + member of the cluster. + :raises RuntimeError: The migration failed, with the platform's cause. + """ + ... + + def set_node_maintenance(self, node: str, enabled: bool) -> None: + """Switch HA maintenance mode of a cluster node; HA guests leave it + while it is on. + + :raises ValueError: If ``node`` is not a member of the cluster. + :raises RuntimeError: If the platform refuses or cannot be asked. + """ + ... + + def get_hypervisor_hosts(self) -> list[HypervisorHostDict]: + """The hosts this hypervisor or its manager runs, and how each stands: + connected, in maintenance, with configuration issues, under a central + manager. Facts; what is a problem is the caller's to decide.""" + ... + + def get_vm_cpu_types(self) -> list[VMCpuTypeDict]: """ List the virtual CPU models a new VM may be given on the node it will be created on. diff --git a/napalm_device_types/interface_filter.py b/napalm_device_types/interface_filter.py index 4360e58..45599d7 100644 --- a/napalm_device_types/interface_filter.py +++ b/napalm_device_types/interface_filter.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Dropping interfaces that are noise rather than signal. An access point exposes radio PHYs and a loopback alongside its real @@ -9,19 +8,19 @@ attributes are the customisation point. from __future__ import annotations -from typing import Any, ClassVar, Dict, FrozenSet, Tuple +from typing import Any, ClassVar class InterfaceFilterMixin: """Adds :meth:`_filter_interfaces` for drivers that report raw interface dicts.""" #: Interface names dropped outright. - _EXCLUDED_INTERFACES: ClassVar[FrozenSet[str]] = frozenset({"lo"}) + _EXCLUDED_INTERFACES: ClassVar[frozenset[str]] = frozenset({"lo"}) #: Name prefixes dropped -- ``phy*`` are radio devices, not links. - _EXCLUDED_INTERFACE_PREFIXES: ClassVar[Tuple[str, ...]] = ("phy",) + _EXCLUDED_INTERFACE_PREFIXES: ClassVar[tuple[str, ...]] = ("phy",) - def _filter_interfaces(self, interfaces: Dict[str, Any]) -> Dict[str, Any]: + def _filter_interfaces(self, interfaces: dict[str, Any]) -> dict[str, Any]: """Remove loopback and radio-device (phy*) interfaces from an interface dict.""" return { name: data diff --git a/napalm_device_types/kernel.py b/napalm_device_types/kernel.py index 463c2c4..629ddab 100644 --- a/napalm_device_types/kernel.py +++ b/napalm_device_types/kernel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """What the running kernel has built and loaded. A kernel CVE's exploitability often hangs on code that is simply not there -- @@ -25,7 +24,7 @@ import base64 import binascii import gzip import zlib -from typing import Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import KernelFactsDict @@ -42,7 +41,8 @@ KERNEL_FACTS_COMMAND = ( "if [ -r $m/modules.builtin ]; then echo '[builtin]'; cat $m/modules.builtin; fi; " "if [ -r $m/modules.dep ]; then echo '[available]'; cut -d: -f1 $m/modules.dep; fi; " "if [ -r /boot/config-$r ]; then echo '[config]'; grep '^CONFIG_' /boot/config-$r; " - "elif [ -r /proc/config.gz ]; then echo '[config]'; zcat /proc/config.gz | grep '^CONFIG_'; fi; " + "elif [ -r /proc/config.gz ]; then echo '[config]'; " + "zcat /proc/config.gz | grep '^CONFIG_'; fi; " "} 2>/dev/null | gzip -c | base64; " "printf '%s%s\\n' KFACTS_ END" ) @@ -78,9 +78,9 @@ def _report(output: str) -> str: return report -def _sections(report: str) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: Optional[List[str]] = None +def _sections(report: str) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] | None = None for line in report.splitlines(): line = line.strip() if line.startswith("[") and line.endswith("]"): @@ -90,8 +90,8 @@ def _sections(report: str) -> Dict[str, List[str]]: return sections -def _config(lines: List[str]) -> Dict[str, str]: - config: Dict[str, str] = {} +def _config(lines: list[str]) -> dict[str, str]: + config: dict[str, str] = {} for line in lines: option, sep, value = line.partition("=") if not sep: @@ -112,7 +112,7 @@ def parse_kernel_facts(output: str) -> KernelFactsDict: """ sections = _sections(_report(output)) - def names(key: str) -> Optional[List[str]]: + def names(key: str) -> list[str] | None: if key not in sections: return None return sorted({module_name(line) for line in sections[key]}) diff --git a/napalm_device_types/lag.py b/napalm_device_types/lag.py index 1d715d0..3252db2 100644 --- a/napalm_device_types/lag.py +++ b/napalm_device_types/lag.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Logical LAG entries for ``get_interfaces()``, built from their member ports. Some switches list only physical ports, each tagged with the trunk it belongs @@ -10,17 +9,17 @@ the same for every vendor, so it lives here once; a driver only has to set from __future__ import annotations import re -from typing import Any, Dict, List, Optional +from typing import Any -def _port_order(name: str) -> List[Any]: +def _port_order(name: str) -> list[Any]: return [int(p) if p.isdigit() else p for p in re.split(r"(\d+)", name)] def add_lag_interfaces( - interfaces: Dict[str, Dict[str, Any]], - lag_modes: Optional[Dict[str, str]] = None, -) -> Dict[str, Dict[str, Any]]: + interfaces: dict[str, dict[str, Any]], + lag_modes: dict[str, str] | None = None, +) -> dict[str, dict[str, Any]]: """Return *interfaces* plus one logical entry per ``trunk_group``. The LAG entry is up/enabled if any member is, its speed is the members' @@ -31,7 +30,7 @@ def add_lag_interfaces( mode gets no ``lag_mode`` key rather than a guessed one. """ result = dict(interfaces) - groups: Dict[str, List[str]] = {} + groups: dict[str, list[str]] = {} for name, iface in interfaces.items(): group = iface.get("trunk_group") if group: @@ -41,7 +40,7 @@ def add_lag_interfaces( if group in result: continue members = sorted(members, key=_port_order) - lag: Dict[str, Any] = { + lag: dict[str, Any] = { "is_up": any(interfaces[m].get("is_up") for m in members), "is_enabled": any(interfaces[m].get("is_enabled") for m in members), "description": f"LAG ({', '.join(members)})", diff --git a/napalm_device_types/licenses.py b/napalm_device_types/licenses.py new file mode 100644 index 0000000..ceeb1ea --- /dev/null +++ b/napalm_device_types/licenses.py @@ -0,0 +1,31 @@ +"""Licenses and subscriptions a device reports. + +A fact for the caller (docs/STRUCTURE.md): whether a free edition, a missing +subscription or a license a feature needs is a problem is decided above the +driver. Declared only; a driver that can read licenses implements it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import LicenseDict + + +class LicenseMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_licenses(self) -> list[LicenseDict]: + """Every license or subscription the device knows of, with its status. + + Example:: + + [{"name": "pve-subscription", "status": "none"}, + {"name": "vSphere", "status": "active", "edition": "Free"}, + {"name": "DCMS", "status": "missing", + "withholds": ["drives", "firmware inventory"]}] + + ``withholds`` names what a device keeps back for want of a license while + the rest of it answers; a missing license without it may keep back all. + """ + ... diff --git a/napalm_device_types/listening.py b/napalm_device_types/listening.py index 73a625e..a622bd9 100644 --- a/napalm_device_types/listening.py +++ b/napalm_device_types/listening.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Listening sockets: what listens on which address, and which service it is. Whether a service is reachable from outside its host is decided by what it @@ -34,7 +33,7 @@ from __future__ import annotations import re from shlex import quote -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import ListeningSocketDict, ListeningSocketsDict from napalm_device_types.terminal import strip_terminal_codes @@ -58,11 +57,11 @@ LISTENING_SOCKETS_COMMAND = ( "t=netstat; s=$(netstat -lntup 2>&1); r=$?; " "pids=$(printf '%s\\n' \"$s\" | grep -o ' [0-9][0-9]*/' | tr -d ' /'); " "else t=; fi; " - "if [ -n \"$t\" ]; then " - "echo \"[$t]\"; printf '%s\\n' \"$s\"; echo \"__SS_RC=$r\"; " + 'if [ -n "$t" ]; then ' + 'echo "[$t]"; printf \'%s\\n\' "$s"; echo "__SS_RC=$r"; ' "echo '[cgroups]'; " "for p in $(printf '%s\\n' \"$pids\" | sort -u); do " - "sed \"s|^|$p |\" /proc/$p/cgroup 2>/dev/null; done; " + 'sed "s|^|$p |" /proc/$p/cgroup 2>/dev/null; done; ' "else echo '[no-ss]'; fi; " "printf '%s%s\\n' SOCK_ END" ) @@ -88,7 +87,7 @@ class ListeningSocketsUnavailable(NotImplementedError): """The host has neither ``ss`` nor ``netstat``; nothing to read, nothing to retry.""" -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -98,9 +97,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: List[str] = [] +def _sections(lines: list[str]) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines: if line.startswith("[") and line.endswith("]") and " " not in line: current = sections.setdefault(line[1:-1], []) @@ -109,7 +108,7 @@ def _sections(lines: List[str]) -> Dict[str, List[str]]: return sections -def _split_local(local: str) -> Optional[Tuple[str, Optional[str], int]]: +def _split_local(local: str) -> tuple[str, str | None, int] | None: """``[fe80::1]%eth0:546`` (or ``[fe80::1%eth0]:546``) -> ``("fe80::1", "eth0", 546)``; None if no port.""" host, sep, port = local.rpartition(":") @@ -122,9 +121,9 @@ def _split_local(local: str) -> Optional[Tuple[str, Optional[str], int]]: return address or "*", zone or None, int(port) -def _cgroup_paths(lines: List[str]) -> Dict[int, str]: +def _cgroup_paths(lines: list[str]) -> dict[int, str]: """Each process's cgroup path: the unified hierarchy, or systemd's under v1.""" - paths: Dict[int, str] = {} + paths: dict[int, str] = {} for line in lines: pid, _, entry = line.partition(" ") parts = entry.split(":", 2) @@ -136,7 +135,7 @@ def _cgroup_paths(lines: List[str]) -> Dict[int, str]: return paths -def _unit(path: Optional[str]) -> Optional[str]: +def _unit(path: str | None) -> str | None: services = _SERVICE_RE.findall(path or "") if services: return str(services[-1]) @@ -144,17 +143,17 @@ def _unit(path: Optional[str]) -> Optional[str]: return str(procd.group(1)) if procd else None -def _container(path: Optional[str]) -> Optional[str]: +def _container(path: str | None) -> str | None: match = _CONTAINER_RE.search(path or "") return (match.group(1) or match.group(2)) if match else None def _entry( proto: str, - local: Tuple[str, Optional[str], int], - process: Optional[str], - pid: Optional[int], - paths: Dict[int, str], + local: tuple[str, str | None, int], + process: str | None, + pid: int | None, + paths: dict[int, str], ) -> ListeningSocketDict: address, interface, port = local path = paths.get(pid) if pid is not None else None @@ -170,7 +169,7 @@ def _entry( } -def _ss_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: +def _ss_socket(line: str, paths: dict[int, str]) -> ListeningSocketDict | None: parts = line.split() if len(parts) < 5 or parts[0] not in _PROTOCOLS: return None @@ -182,7 +181,7 @@ def _ss_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict return _entry(parts[0], local, process, pid, paths) -def _netstat_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocketDict]: +def _netstat_socket(line: str, paths: dict[int, str]) -> ListeningSocketDict | None: """``Proto Recv-Q Send-Q Local Foreign [State] PID/Program`` -- a UDP line has no state, and ``-`` is a socket without a process.""" parts = line.split() @@ -204,7 +203,7 @@ def _netstat_socket(line: str, paths: Dict[int, str]) -> Optional[ListeningSocke _PARSERS = {"ss": _ss_socket, "netstat": _netstat_socket} -def parse_listening_sockets(output: str) -> List[ListeningSocketDict]: +def parse_listening_sockets(output: str) -> list[ListeningSocketDict]: """Parse what :data:`LISTENING_SOCKETS_COMMAND` printed, sorted by protocol, port and address. @@ -223,7 +222,9 @@ def parse_listening_sockets(output: str) -> List[ListeningSocketDict]: paths = _cgroup_paths(sections.get("cgroups", [])) parse = _PARSERS[tool] sockets = [s for s in (parse(line, paths) for line in lines) if s is not None] - return sorted(sockets, key=lambda s: (s["proto"], s["port"], s["address"], s["interface"] or "")) + return sorted( + sockets, key=lambda s: (s["proto"], s["port"], s["address"], s["interface"] or "") + ) class ListeningSocketsMixin: diff --git a/napalm_device_types/local_accounts.py b/napalm_device_types/local_accounts.py new file mode 100644 index 0000000..d7f7b1e --- /dev/null +++ b/napalm_device_types/local_accounts.py @@ -0,0 +1,751 @@ +"""Personal login accounts on a host: reading them, adding one, locking one again. + +A person who is to work on a host gets an account of their own: they log in +with their public key and, where they are to administer the host, may use +sudo with their own password. When they leave, the account is locked, not +deleted: their key goes, sudo goes, the login is refused, and the home stays. + +:class:`LocalAccountsMixin` holds what is the same on every POSIX host -- the +checks, reading an account back, the order of the steps, the key file and the +sudo rule. A driver supplies the commands that differ between operating +systems: creating, opening and locking an account, reading its lock, and +taking it out of a group. + +**Every command root runs is one fixed command**, with no shell syntax of its +own -- ``useradd ...``, ``install -m 440 ...``, ``rm -f -- `` -- so a +sudoers allowlist of commands can name each one. What goes into a file is +staged first, without privilege; root never reads stdin, where a sudo that +does not prompt would leave the password. + +**Nothing root runs follows the person's files.** The key files live in a home +the person owns, where a symbolic link may point anywhere, so they are read and +written as that person (``sudo -n -u ``). Only the existence of a key +file is asked as root. + +**The sudo rule is a file of its own** in the drop-in directory sudo includes, +`` ALL=(ALL) ALL``: the person's own password, as on a new VM. A group +alone is not enough, as some distributions grant ``%wheel`` and others +``%sudo``. ``visudo`` checks the rule before it is installed, and ``sudo -l`` +afterwards shows whether it took effect. A rule elsewhere -- cloud-init's +``90-cloud-init-users`` -- is not edited; it is reported. + +**The login user's own account and system accounts are never touched**: +locking the first would lock the caller out. + +**netOrk's own account** (``add_service_account``) is made the same way, +through the account it logs in with today: a key and no password, and a rule +of its own -- `` ALL=(ALL) NOPASSWD: ALL`` in ``85-service-`` -- +so it gains root without a password that could end up anywhere. Once netOrk +logs in with it, the person's account it used before is one like any other. +""" + +from __future__ import annotations + +import re +import shlex +from collections.abc import Iterable, Sequence +from typing import TYPE_CHECKING, ClassVar + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepDict, FixStepOutcomeDict + +if TYPE_CHECKING: # pragma: no cover - for type checkers only + from napalm_device_types.channel import CommandResult + +#: A login name every POSIX host takes as it is: lowercase, a letter or an +#: underscore first, at most 32 characters. It is also safe in a root command. +ACCOUNT_NAME_RE = re.compile(r"[a-z_][a-z0-9_.-]{0,31}") + +#: The crypt(3) prefixes of each password scheme a driver may take. +PASSWORD_SCHEMES: dict[str, tuple[str, ...]] = { + "sha512": ("$6$",), + "bcrypt": ("$2b$", "$2a$", "$2y$"), +} +_CRYPT_RE = re.compile(r"\$[0-9a-z]+\$[./0-9A-Za-z$]+") + +#: The public key types OpenSSH takes in a key file. +_KEY_TYPE_RE = re.compile( + r"(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp(256|384|521)" + r"|sk-ssh-ed25519@openssh\.com|sk-ecdsa-sha2-nistp256@openssh\.com)" +) +_KEY_BLOB_RE = re.compile(r"[A-Za-z0-9+/]+={0,3}") + +#: Before the rule's file name; after cloud-init's ``90-`` it would win, so before it. +SUDOERS_FILE_PREFIX = "80-account-" +#: Before the file name of netOrk's own account's rule; after a person's, before cloud-init's. +SERVICE_SUDOERS_FILE_PREFIX = "85-service-" + +#: A line of ``sudo -l`` that lets the account run every command as root. +_RUN_ALL_RE = re.compile(r"^\s*\(ALL(\s*:\s*ALL)?\)\s+([A-Z_]+:\s*)*ALL\s*$", re.MULTILINE) +#: The same, without a password. +_RUN_ALL_NOPASSWD_RE = re.compile( + r"^\s*\(ALL(\s*:\s*ALL)?\)\s+([A-Z_]+:\s*)*NOPASSWD:\s*([A-Z_]+:\s*)*ALL\s*$", + re.MULTILINE, +) + +#: Login shells that refuse every login: an account with one belongs to a service. +_NO_LOGIN_SHELLS = frozenset({"nologin", "false"}) + +#: Stages stdin as a file anyone may read -- the person reads the key file +#: from it -- and prints its path. The content is a public key or a sudo rule. +_STAGE = 't=$(mktemp) && chmod 644 "$t" && cat > "$t" && printf %s "$t"' + + +class LocalAccountDict(TypedDict): + """A person's account as the host has it (``get_local_accounts``).""" + + name: str + uid: int + home: str + shell: str + #: Primary group first. + groups: list[str] + #: No login, with a key or a password. + locked: bool + #: ``sudo -l`` lets it run every command as root. + admin: bool + #: The lines of its key files; None when they could not be read (a locked + #: account cannot run a command any more). + authorized_keys: list[str] | None + + +class LocalAccountChangeDict(FixResultDict): + """The result of ``add_local_account`` and ``lock_local_account``.""" + + #: The account read back after the change; None when it is not there or + #: could not be read. + account: LocalAccountDict | None + + +def sudoers_rule(name: str) -> str: + """The rule that lets *name* run everything as root, with its own password.""" + return f"{name} ALL=(ALL) ALL\n" + + +def sudoers_rule_path(directory: str, name: str) -> str: + """Where *name*'s rule lives in *directory*. + + sudo skips a file whose name holds a dot, so a dot becomes ``_2e``, and an + underscore ``__``: no two names share a file. + """ + return f"{directory}/{SUDOERS_FILE_PREFIX}{_rule_file_name(name)}" + + +def service_sudoers_rule(name: str) -> str: + """The rule that lets netOrk's own account *name* run everything as root, no password.""" + return f"{name} ALL=(ALL) NOPASSWD: ALL\n" + + +def service_sudoers_rule_path(directory: str, name: str) -> str: + """Where the rule of netOrk's own account *name* lives in *directory*.""" + return f"{directory}/{SERVICE_SUDOERS_FILE_PREFIX}{_rule_file_name(name)}" + + +def _rule_file_name(name: str) -> str: + return name.replace("_", "__").replace(".", "_2e") + + +def authorized_keys_paths(home: str) -> tuple[str, str]: + """The key files sshd reads in *home* unless configured otherwise.""" + return f"{home}/.ssh/authorized_keys", f"{home}/.ssh/authorized_keys2" + + +def check_account_name(name: str) -> None: + """:raises ValueError: when *name* is no login name every host takes.""" + if not ACCOUNT_NAME_RE.fullmatch(name): + raise ValueError(f"not an account name: {name!r}") + + +def _key_blob(line: str) -> str | None: + """The base64 part of a key file line, options and comment aside.""" + words = line.split() + for i, word in enumerate(words[:-1]): + if _KEY_TYPE_RE.fullmatch(word): + return words[i + 1] + return None + + +def check_public_key(key: str) -> None: + """:raises ValueError: when *key* is not one OpenSSH public key on one line.""" + words = key.split() + if ( + "\n" in key.strip() + or len(words) < 2 + or not _KEY_TYPE_RE.fullmatch(words[0]) + or not _KEY_BLOB_RE.fullmatch(words[1]) + ): + raise ValueError("not a public key: expected ' [comment]' on one line") + + +def _command(argv: Sequence[str]) -> str: + return shlex.join(argv) + + +def _as(name: str, *argv: str) -> str: + """*argv* run as *name*, from root: nothing it does reaches beyond that account.""" + return _command(["sudo", "-n", "-u", name, *argv]) + + +def _said(result: CommandResult) -> str: + return (result.stderr or result.stdout).strip() or f"exit {result.exit_code}" + + +class LocalAccountsMixin: + """Reads, adds and locks personal accounts; a driver supplies the OS's commands. + + Mixed in by the drivers that can, so ``callable(getattr(driver, + "add_local_account", None))`` says whether a driver can. + """ + + #: The login shell of a new account. + ACCOUNT_SHELL: ClassVar[str] = "/bin/sh" + #: The password scheme the host's accounts take, a key of :data:`PASSWORD_SCHEMES`. + ACCOUNT_PASSWORD_SCHEME: ClassVar[str] = "sha512" + #: Groups that make an account an administrator; a person without sudo leaves them. + ACCOUNT_ADMIN_GROUPS: ClassVar[tuple[str, ...]] = ("wheel",) + #: The lowest uid of a person's account; below are the system's. + ACCOUNT_MIN_UID: ClassVar[int] = 1000 + #: The drop-in directory sudo includes. + SUDOERS_DIR: ClassVar[str] = "/etc/sudoers.d" + #: The group of root's files. + ROOT_GROUP: ClassVar[str] = "root" + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: ... + + # -- what a driver supplies ------------------------------------------------------ + # + # ``_account_*`` are the driver's: the five below it must supply, the key + # store (``_account_key_files``, ``_account_read_keys``, ``_account_add_key``, + # ``_account_remove_keys``) it may replace. ``_accounts_*`` are this mixin's own. + + def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: + """The commands that create *name* with a home and :attr:`ACCOUNT_SHELL`. + + Without *password_hash* the account takes no password, only a key. + """ + raise NotImplementedError + + def _account_open( + self, name: str, password_hash: str | None, *, locked: bool + ) -> list[list[str]]: + """The commands that let *name* log in again, with *password_hash* if given.""" + raise NotImplementedError + + def _account_lock(self, name: str, *, locked: bool) -> list[list[str]]: + """The commands that refuse *name* every login, a key's too; none when *locked*.""" + raise NotImplementedError + + def _account_locked(self, name: str, passwd: list[str]) -> bool: + """Whether *name* is locked; *passwd* is its ``getent passwd`` line, split. + + :raises RuntimeError: when that cannot be read. + """ + raise NotImplementedError + + def _account_leave_group(self, name: str, group: str, groups: list[str]) -> list[str]: + """The command that takes *name* out of *group*; *groups* are all of its own.""" + raise NotImplementedError + + def _account_create_with_uid(self, name: str, uid: int) -> list[list[str]]: + """The commands that create *name* as :meth:`_account_create` does, without a + password, and with the uid *uid*. + + Only netOrk's own account asks for one (``add_service_account(uid=)``). + """ + raise NotImplementedError + + # -- reading --------------------------------------------------------------------- + + def get_local_accounts(self, names: Iterable[str]) -> list[LocalAccountDict]: + """The accounts of *names* the host has, in that order; a missing one is left out. + + :raises ValueError: for a name that is no account name. + :raises RuntimeError: when an account cannot be read. + """ + wanted = list(names) + for name in wanted: + check_account_name(name) + found = (self._accounts_read(name) for name in wanted) + return [account for account in found if account is not None] + + def _accounts_passwd(self, name: str) -> list[str] | None: + result = self.run_command(_command(["getent", "passwd", name]), timeout=30) + if result.exit_code == 2: + return None + fields = result.stdout.strip().split(":") + if result.exit_code != 0 or len(fields) < 7 or fields[0] != name: + raise RuntimeError(f"cannot read the account {name}: {_said(result)}") + return fields + + def _accounts_read(self, name: str) -> LocalAccountDict | None: + passwd = self._accounts_passwd(name) + if passwd is None: + return None + groups = self.run_command(_command(["id", "-Gn", name]), timeout=30) + if groups.exit_code != 0: + raise RuntimeError(f"cannot read the groups of {name}: {_said(groups)}") + home = passwd[5] + return { + "name": name, + "uid": int(passwd[2]), + "home": home, + "shell": passwd[6], + "groups": groups.stdout.split(), + "locked": self._account_locked(name, passwd), + "admin": self._accounts_admin(name), + "authorized_keys": self._account_read_keys(name, home), + } + + def _accounts_admin(self, name: str) -> bool: + result = self.run_command(_command(["sudo", "-l", "-U", name]), privileged=True, timeout=30) + if result.exit_code != 0: + raise RuntimeError(f"cannot ask sudo about {name}: {_said(result)}") + return bool(_RUN_ALL_RE.search(result.stdout)) + + def _account_key_files(self, name: str, home: str) -> tuple[str, ...]: + """Every file sshd reads *name*'s keys from; a host that adds one of its own says so.""" + return authorized_keys_paths(home) + + def _account_read_keys(self, name: str, home: str) -> list[str] | None: + """The lines of :meth:`_account_key_files`, read as *name*; None when it cannot.""" + lines: list[str] = [] + for path in self._account_key_files(name, home): + there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) + if there.exit_code != 0: + continue + result = self.run_command(_as(name, "cat", "--", path), privileged=True, timeout=30) + if result.exit_code != 0: + return None + lines += [ln for ln in result.stdout.splitlines() if ln.strip()] + return lines + + # -- changing ---------------------------------------------------------------------- + + def add_local_account( + self, + name: str, + public_key: str, + *, + password_hash: str | None = None, + admin: bool = False, + ) -> LocalAccountChangeDict: + """Give *name* an account that logs in with *public_key*, or open it again. + + An account that is there is opened again and keeps its other keys; a + new one gets a home. *password_hash* (crypt(3), in + :attr:`ACCOUNT_PASSWORD_SCHEME`) becomes its password; *admin* gives it + the sudo rule, which asks for that password -- so an administrator + needs one. Without *admin*, the rule and :attr:`ACCOUNT_ADMIN_GROUPS` + are taken away. + + Steps: ``account``, then ``sudo`` and ``keys``, each of which runs when + the other failed. + + :raises ValueError: for a name, key or hash no host takes, the login + user's own account, or a system account. + """ + check_account_name(name) + check_public_key(public_key) + if password_hash is not None: + self._accounts_check_hash(password_hash) + existing = self._accounts_changeable(name) + first = run_fix_steps( + [("account", lambda: self._accounts_open_or_create(name, existing, password_hash))] + ) + parts = [first] + if first["success"]: + parts += [ + run_fix_steps([("sudo", lambda: self._accounts_set_sudo(name, admin=admin))]), + run_fix_steps([("keys", lambda: self._accounts_add_key(name, public_key))]), + ] + return self._accounts_result(name, parts) + + def lock_local_account(self, name: str) -> LocalAccountChangeDict: + """Lock *name*: its keys go, every login is refused, sudo goes; its home stays. + + Steps ``keys``, ``lock`` and ``sudo``, each run when one before failed: + the lock matters most. A sudo rule outside the drop-in directory is + reported, not edited -- with the account locked it can no longer be used. + + :raises ValueError: for a name no host takes, the login user's own + account, or a system account. + """ + check_account_name(name) + if self._accounts_changeable(name) is None: + step: FixStepDict = {"step": "account", "ok": False, "output": f"no account {name}"} + return {**_joined([step]), "account": None} + parts = [ + run_fix_steps([("keys", lambda: self._accounts_remove_keys(name))]), + run_fix_steps([("lock", lambda: self._accounts_lock(name))]), + run_fix_steps( + [("sudo", lambda: self._accounts_set_sudo(name, admin=False, locked=True))] + ), + ] + return self._accounts_result(name, parts) + + def add_service_account( + self, name: str, public_key: str, *, uid: int | None = None + ) -> LocalAccountChangeDict: + """Make *name* netOrk's own account: it logs in with *public_key*, and gains + root without a password. + + Created through the account the driver logs in with now, or opened + again when it is there; its other keys stay. It takes no password. Run + again with the account the driver already logs in with, it only adds + what is missing. + + A new account gets *uid* when one is given -- the same on every host -- + once no other account has it; an account that is there keeps its own, + as a uid is never changed under files that carry it. + + Steps as in :meth:`add_local_account`: ``account``, then ``sudo`` and + ``keys``. The new login itself is for the caller to try. + + An account that is there and cannot log in (``nologin``, ``false``) is + another service's -- a satellite's ``netork``, say -- and is left alone. + + :raises ValueError: for a name or key no host takes, root, a uid of 0 or + less, or another service's account. + :raises RuntimeError: when the login user cannot be told. + """ + check_public_key(public_key) + login, existing = self._accounts_service_target(name, uid) + if name == login: + step: FixStepDict = { + "step": "account", + "ok": True, + "output": f"this driver logs in as {name} already", + } + first: FixResultDict = _joined([step]) + else: + first = run_fix_steps( + [("account", lambda: self._accounts_service_account(name, existing, uid))] + ) + parts = [first] + if first["success"]: + parts += [ + run_fix_steps([("sudo", lambda: self._accounts_grant_service_sudo(name))]), + run_fix_steps([("keys", lambda: self._accounts_add_key(name, public_key))]), + ] + return self._accounts_result(name, parts) + + def _accounts_service_target(self, name: str, uid: int | None) -> tuple[str, list[str] | None]: + """The login user and *name*'s passwd line, once *name* may become netOrk's. + + :raises ValueError: as :meth:`add_service_account` says. + """ + check_account_name(name) + if uid is not None and uid <= 0: + raise ValueError(f"not a uid for netOrk's own account: {uid}") + login = self._accounts_login() + existing = self._accounts_passwd(name) + if existing is not None and int(existing[2]) == 0: + raise ValueError(f"{name} is root") + if existing is not None and existing[6].rsplit("/", 1)[-1] in _NO_LOGIN_SHELLS: + raise ValueError( + f"{name} is there and cannot log in ({existing[6]}): another service's account" + ) + return login, existing + + def _accounts_service_account( + self, name: str, passwd: list[str] | None, uid: int | None + ) -> FixStepOutcomeDict: + """netOrk's own account opened or made, a new one with *uid* when given.""" + if uid is None: + return self._accounts_open_or_create(name, passwd, None) + if passwd is not None: + outcome = self._accounts_open_or_create(name, passwd, None) + if outcome["ok"] and int(passwd[2]) != uid: + outcome["output"] += f"; it keeps its uid {passwd[2]}, not {uid}" + return outcome + owner = self._accounts_passwd_of_uid(uid) + if owner is not None: + return {"ok": False, "output": f"uid {uid} is the account {owner}'s"} + try: + create = self._account_create_with_uid(name, uid) + except NotImplementedError: + return {"ok": False, "output": f"this driver cannot give an account the uid {uid}"} + failed = self._accounts_run(create) + if failed is not None: + return {"ok": False, "output": _said(failed)} + return {"ok": True, "output": f"account {name} created, uid {uid}"} + + def _accounts_passwd_of_uid(self, uid: int) -> str | None: + """The name of the account with *uid*, or None.""" + result = self.run_command(_command(["getent", "passwd", str(uid)]), timeout=30) + if result.exit_code == 2: + return None + if result.exit_code != 0: + raise RuntimeError(f"cannot ask who has uid {uid}: {_said(result)}") + return result.stdout.split(":", 1)[0].strip() or None + + def _accounts_check_hash(self, password_hash: str) -> None: + prefixes = PASSWORD_SCHEMES[self.ACCOUNT_PASSWORD_SCHEME] + if not _CRYPT_RE.fullmatch(password_hash) or not password_hash.startswith(prefixes): + raise ValueError(f"not a {self.ACCOUNT_PASSWORD_SCHEME} password hash") + + def _accounts_changeable(self, name: str) -> list[str] | None: + """*name*'s passwd line, None when there is none. + + :raises ValueError: for the login user's own account or a system account. + :raises RuntimeError: when the login user cannot be told. + """ + if name == self._accounts_login(): + raise ValueError(f"{name} is the account this driver logs in with") + passwd = self._accounts_passwd(name) + if passwd is not None and int(passwd[2]) < self.ACCOUNT_MIN_UID: + raise ValueError(f"{name} is a system account (uid {passwd[2]})") + return passwd + + def _accounts_login(self) -> str: + """:raises RuntimeError: when the login user cannot be told.""" + login = self.run_command("id -un", timeout=30).stdout.strip() + if not login: + raise RuntimeError("cannot tell which account this driver logs in with") + return login + + def _accounts_result(self, name: str, parts: list[FixResultDict]) -> LocalAccountChangeDict: + steps = [s for part in parts for s in part["steps"]] + result = _joined(steps) + account: LocalAccountDict | None = None + try: + account = self._accounts_read(name) + except RuntimeError as exc: + result["output"] += f"\n[read back] {exc}" + return {**result, "account": account} + + def _accounts_root(self, argv: Sequence[str]) -> CommandResult: + return self.run_command(_command(argv), privileged=True, timeout=60) + + def _accounts_run(self, commands: list[list[str]]) -> CommandResult | None: + """Run *commands* as root in order; the first that failed, or None.""" + for argv in commands: + result = self._accounts_root(argv) + if result.exit_code != 0: + return result + return None + + def _accounts_open_or_create( + self, name: str, passwd: list[str] | None, password_hash: str | None + ) -> FixStepOutcomeDict: + if passwd is None: + failed = self._accounts_run(self._account_create(name, password_hash)) + done = f"account {name} created" + else: + locked = self._account_locked(name, passwd) + failed = self._accounts_run(self._account_open(name, password_hash, locked=locked)) + done = f"account {name} opened again" if locked else f"account {name} is there" + if failed is not None: + return {"ok": False, "output": _said(failed)} + return {"ok": True, "output": done + (", password set" if password_hash else "")} + + def _accounts_lock(self, name: str) -> FixStepOutcomeDict: + passwd = self._accounts_passwd(name) + if passwd is None: + return {"ok": False, "output": f"no account {name}"} + locked = self._account_locked(name, passwd) + failed = self._accounts_run(self._account_lock(name, locked=locked)) + if failed is not None: + return {"ok": False, "output": _said(failed)} + if not self._account_locked(name, passwd): + return {"ok": False, "output": f"{name} is still not locked"} + return {"ok": True, "output": f"{name} was locked already" if locked else f"{name} locked"} + + # -- sudo -------------------------------------------------------------------------- + + def _accounts_set_sudo( + self, name: str, *, admin: bool, locked: bool = False + ) -> FixStepOutcomeDict: + path = sudoers_rule_path(self.SUDOERS_DIR, name) + if admin: + return self._accounts_grant_sudo(name, path) + said: list[str] = [] + removed = self._accounts_root(["rm", "-f", "--", path]) + if removed.exit_code != 0: + return {"ok": False, "output": f"{path}: {_said(removed)}"} + groups = self.run_command(_command(["id", "-Gn", name]), timeout=30).stdout.split() + for group in self.ACCOUNT_ADMIN_GROUPS: + if group not in groups[1:]: + continue + left = self._accounts_root(self._account_leave_group(name, group, groups)) + if left.exit_code != 0: + return {"ok": False, "output": f"cannot take {name} out of {group}: {_said(left)}"} + groups.remove(group) + said.append(f"{name} left {group}") + if self._accounts_admin(name): + rest = f"another rule than {path} still lets {name} run everything as root" + if locked: + return {"ok": True, "output": f"{rest}; the account is locked, so it cannot use it"} + return {"ok": False, "output": rest} + said.append(f"{name} has no sudo") + return {"ok": True, "output": "\n".join(said)} + + def _accounts_grant_sudo(self, name: str, path: str) -> FixStepOutcomeDict: + failed = self._accounts_install_rule(sudoers_rule(name), path) + if failed is not None: + return failed + if not self._accounts_admin(name): + return { + "ok": False, + "output": f"{path} is written, but sudo does not read it: " + f"does sudoers include {self.SUDOERS_DIR}?", + } + return {"ok": True, "output": f"{name} may use sudo with its password ({path})"} + + def _accounts_grant_service_sudo(self, name: str) -> FixStepOutcomeDict: + path = service_sudoers_rule_path(self.SUDOERS_DIR, name) + failed = self._accounts_install_rule(service_sudoers_rule(name), path) + if failed is not None: + return failed + result = self.run_command(_command(["sudo", "-l", "-U", name]), privileged=True, timeout=30) + if result.exit_code != 0: + return {"ok": False, "output": f"cannot ask sudo about {name}: {_said(result)}"} + if not _RUN_ALL_NOPASSWD_RE.search(result.stdout): + return { + "ok": False, + "output": f"{path} is written, but sudo does not read it: " + f"does sudoers include {self.SUDOERS_DIR}?", + } + return {"ok": True, "output": f"{name} may use sudo without a password ({path})"} + + def _accounts_install_rule(self, rule: str, path: str) -> FixStepOutcomeDict | None: + """Check *rule* with visudo and install it at *path*; the failure, or None.""" + staged = self._accounts_stage(rule) + try: + checked = self._accounts_root(["visudo", "-cf", staged]) + if checked.exit_code != 0: + return {"ok": False, "output": f"visudo refuses the rule: {_said(checked)}"} + installed = self._accounts_root( + ["install", "-m", "440", "-o", "root", "-g", self.ROOT_GROUP, "--", staged, path] + ) + if installed.exit_code != 0: + return {"ok": False, "output": f"{path}: {_said(installed)}"} + finally: + self._accounts_unstage(staged) + return None + + # -- keys -------------------------------------------------------------------------- + + def _accounts_stage(self, content: str) -> str: + """*content* in a fresh file of the login user's; its path. + + :raises RuntimeError: when it cannot be staged. + """ + result = self.run_command(_STAGE, timeout=30, stdin=content.encode()) + path = result.stdout.strip().splitlines()[-1:] or [""] + if result.exit_code != 0 or not path[0].startswith("/"): + raise RuntimeError(f"cannot stage a file: {_said(result)}") + return path[0] + + def _accounts_unstage(self, path: str) -> None: + self.run_command(_command(["rm", "-f", "--", path]), timeout=30) + + def _accounts_home(self, name: str) -> str | None: + passwd = self._accounts_passwd(name) + return passwd[5] if passwd is not None else None + + def _accounts_add_key(self, name: str, public_key: str) -> FixStepOutcomeDict: + home = self._accounts_home(name) + if home is None: + return {"ok": False, "output": f"no account {name}"} + return self._account_add_key(name, home, public_key) + + def _accounts_remove_keys(self, name: str) -> FixStepOutcomeDict: + home = self._accounts_home(name) + if home is None: + return {"ok": False, "output": f"no account {name}"} + return self._account_remove_keys(name, home) + + def _account_add_key(self, name: str, home: str, public_key: str) -> FixStepOutcomeDict: + """Add *public_key* to *name*'s first key file, as *name*; the others stay. + + A driver whose host keeps keys elsewhere overrides this, with + :meth:`_account_read_keys` and :meth:`_account_remove_keys`. + """ + path = authorized_keys_paths(home)[0] + current = self._account_read_keys(name, home) + if current is None: + return {"ok": False, "output": f"cannot read the key files of {name}"} + blob = _key_blob(public_key) + if any(_key_blob(line) == blob for line in current): + return {"ok": True, "output": f"the key is already in {path}"} + mine = self._accounts_key_file(name, path) + if mine is None: + return {"ok": False, "output": f"cannot read {path}"} + staged = self._accounts_stage("".join(f"{line}\n" for line in [*mine, public_key.strip()])) + try: + failed = self._accounts_run_as( + name, + [ + ["mkdir", "-p", "-m", "700", "--", f"{home}/.ssh"], + ["install", "-m", "600", "--", staged, path], + ], + ) + finally: + self._accounts_unstage(staged) + if failed is not None: + return {"ok": False, "output": f"{path}: {_said(failed)}"} + return {"ok": True, "output": f"key added to {path}"} + + def _accounts_key_file(self, name: str, path: str) -> list[str] | None: + there = self.run_command(_command(["test", "-e", path]), privileged=True, timeout=30) + if there.exit_code != 0: + return [] + result = self.run_command(_as(name, "cat", "--", path), privileged=True, timeout=30) + if result.exit_code != 0: + return None + return [line for line in result.stdout.splitlines() if line.strip()] + + def _accounts_run_as(self, name: str, commands: list[list[str]]) -> CommandResult | None: + for argv in commands: + result = self.run_command(_as(name, *argv), privileged=True, timeout=60) + if result.exit_code != 0: + return result + return None + + def _account_remove_keys(self, name: str, home: str) -> FixStepOutcomeDict: + """Remove *name*'s key files in *home*, as *name*; then no key may be left.""" + paths = authorized_keys_paths(home) + failed = self._accounts_run_as(name, [["rm", "-f", "--", *paths]]) + if failed is not None: + return {"ok": False, "output": f"cannot remove the keys of {name}: {_said(failed)}"} + if self._account_read_keys(name, home): + return {"ok": False, "output": f"{name} still has keys"} + return {"ok": True, "output": f"keys of {name} removed"} + + +def _joined(steps: list[FixStepDict]) -> FixResultDict: + return { + "success": bool(steps) and all(s["ok"] for s in steps), + "steps": steps, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in steps), + } + + +__all__ = [ + "ACCOUNT_NAME_RE", + "PASSWORD_SCHEMES", + "SERVICE_SUDOERS_FILE_PREFIX", + "SUDOERS_FILE_PREFIX", + "LocalAccountChangeDict", + "LocalAccountDict", + "LocalAccountsMixin", + "authorized_keys_paths", + "check_account_name", + "check_public_key", + "service_sudoers_rule", + "service_sudoers_rule_path", + "sudoers_rule", + "sudoers_rule_path", +] diff --git a/napalm_device_types/mac_acl.py b/napalm_device_types/mac_acl.py index a84e3cb..b1d20f9 100644 --- a/napalm_device_types/mac_acl.py +++ b/napalm_device_types/mac_acl.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """MAC-based access control, per SSID on an AP and per port on a switch. The reader is identical either way; only the writer is AP-specific so far. @@ -10,7 +9,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import MACACLDict @@ -18,7 +17,7 @@ from napalm_device_types.models import MACACLDict class MacAclMixin: if TYPE_CHECKING: - def get_mac_acl(self) -> Dict[str, MACACLDict]: + def get_mac_acl(self) -> dict[str, MACACLDict]: """ Returns the MAC-address-based access control lists configured per SSID. @@ -44,22 +43,25 @@ class MacAclMixin: "name": "CorpWiFi", "policy": "allow", "entries": [ - {"mac": "AA:BB:CC:DD:EE:01", "action": "allow", "description": "CEO-Laptop"}, - {"mac": "AA:BB:CC:DD:EE:02", "action": "allow", "description": "CFO-Laptop"}, + {"mac": "AA:BB:CC:DD:EE:01", "action": "allow", + "description": "CEO-Laptop"}, + {"mac": "AA:BB:CC:DD:EE:02", "action": "allow", + "description": "CFO-Laptop"}, ], }, "GuestNet": { "name": "GuestNet", "policy": "deny", "entries": [ - {"mac": "DE:AD:BE:EF:00:01", "action": "deny", "description": "blocked device"}, + {"mac": "DE:AD:BE:EF:00:01", "action": "deny", + "description": "blocked device"}, ], }, } """ ... - def push_mac_acl(self, ssid_name: str, mode: str, macs: List[str]) -> None: + def push_mac_acl(self, ssid_name: str, mode: str, macs: list[str]) -> None: """ Rewrites the MAC-address access control list for a single SSID. @@ -73,7 +75,8 @@ class MacAclMixin: Example:: - driver.push_mac_acl("CorpWiFi", "whitelist", ["AA:BB:CC:DD:EE:01", "AA:BB:CC:DD:EE:02"]) + macs = ["AA:BB:CC:DD:EE:01", "AA:BB:CC:DD:EE:02"] + driver.push_mac_acl("CorpWiFi", "whitelist", macs) driver.push_mac_acl("GuestNet", "off", []) """ ... diff --git a/napalm_device_types/management_controller.py b/napalm_device_types/management_controller.py index a1d6df4..e9a9de5 100644 --- a/napalm_device_types/management_controller.py +++ b/napalm_device_types/management_controller.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for server baseboard management controllers (BMCs). @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from napalm_device_types.base import DeviceTypeDriver @@ -107,7 +106,7 @@ class ManagementControllerDriver(DeviceTypeDriver): """ ... - def get_event_log(self, limit: int = 500) -> List[EventLogEntryDict]: + def get_event_log(self, limit: int = 500) -> list[EventLogEntryDict]: """ Returns the BMC's system event log (SEL, Lifecycle Log, IML), newest first. @@ -116,7 +115,7 @@ class ManagementControllerDriver(DeviceTypeDriver): """ ... - def get_firmware_inventory(self) -> List[FirmwareDict]: + def get_firmware_inventory(self) -> list[FirmwareDict]: """ Returns every firmware component the BMC reports: its own, the BIOS, storage controllers, NICs, power supplies. diff --git a/napalm_device_types/media.py b/napalm_device_types/media.py index 20c52f9..ce0882e 100644 --- a/napalm_device_types/media.py +++ b/napalm_device_types/media.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for networked media players. @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver @@ -34,7 +33,7 @@ class MediaDriver(DeviceTypeDriver): if TYPE_CHECKING: - def get_playback_state(self) -> Dict[str, Any]: + def get_playback_state(self) -> dict[str, Any]: """ Returns the transport state of the renderer. @@ -51,7 +50,7 @@ class MediaDriver(DeviceTypeDriver): """Sets the output volume, 0-100.""" ... - def get_zone_info(self) -> List[Dict[str, Any]]: + def get_zone_info(self) -> list[dict[str, Any]]: """ Returns the zones or groups this renderer participates in. diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index cc5e41c..b791092 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -5,9 +5,15 @@ These supplement the types defined in napalm.base.models and are used by the abstract device-type driver classes in this package. """ -from typing import Dict, List, Literal, Optional, Tuple, get_args -from typing_extensions import NotRequired, TypedDict +from typing import Literal, NotRequired, get_args +from typing_extensions import TypedDict + +# The SNMP health profile shapes live in snmp_health_profile.py; re-exported here, where they were. +from napalm_device_types.snmp_health_profile import SnmpHealthProfile as SnmpHealthProfile +from napalm_device_types.snmp_health_profile import SnmpInterfacesSpec as SnmpInterfacesSpec +from napalm_device_types.snmp_health_profile import SnmpMemorySpec as SnmpMemorySpec +from napalm_device_types.snmp_health_profile import SnmpValueSpec as SnmpValueSpec # --------------------------------------------------------------------------- # Common (shared across device types) @@ -36,7 +42,7 @@ class MACACLDict(TypedDict): name: str policy: str - entries: List[MACACLEntryDict] + entries: list[MACACLEntryDict] class PackageDict(TypedDict): @@ -70,9 +76,9 @@ class UpdateDict(TypedDict): current_version: str new_version: str #: Where the new version comes from, e.g. apt's suites "noble-updates,noble-security". - origin: NotRequired[Optional[str]] + origin: NotRequired[str | None] #: True for a security update, False for a known other one, None when unknown. - security: NotRequired[Optional[bool]] + security: NotRequired[bool | None] class InterruptedPackageDict(TypedDict): @@ -100,24 +106,24 @@ class HostStatusDict(TypedDict): """What a host says about its own patch state (``HostStatusMixin.get_host_status``).""" #: True when the host needs a reboot to finish an update, None when it cannot tell. - reboot_required: Optional[bool] + reboot_required: bool | None #: Why, e.g. "kernel 6.8.0-142-generic installed, 6.8.0-139-generic running". - reboot_reason: Optional[str] + reboot_reason: str | None #: True when the host installs updates on its own (unattended-upgrades, dnf-automatic). - auto_updates: Optional[bool] + auto_updates: bool | None #: True when a package manager run was cut off and has to be finished #: (``dpkg --configure -a``) before anything else installs; None when it #: cannot tell, or a package manager is running right now. - package_manager_interrupted: NotRequired[Optional[bool]] + package_manager_interrupted: NotRequired[bool | None] #: The packages that run left halfway; empty when only its journal says so. - interrupted_packages: NotRequired[List[InterruptedPackageDict]] + interrupted_packages: NotRequired[list[InterruptedPackageDict]] #: When the host's own automatic updates last ran successfully (Unix time). - auto_updates_last_success: NotRequired[Optional[int]] + auto_updates_last_success: NotRequired[int | None] #: How often they are meant to run, in seconds; 0 for every time the timer fires. - auto_updates_interval: NotRequired[Optional[int]] + auto_updates_interval: NotRequired[int | None] #: How full ``/``, ``/boot`` and ``/var`` are, each filesystem once; None when #: it could not be read. - filesystems: NotRequired[Optional[List[FilesystemUsageDict]]] + filesystems: NotRequired[list[FilesystemUsageDict] | None] # --------------------------------------------------------------------------- @@ -163,11 +169,11 @@ class ChannelScanEntryDict(TypedDict): bssid: str ssid: str - frequency: int # centre frequency in MHz, e.g. 2437 - channel: int # channel number, e.g. 6 - signal_dbm: int # RSSI in dBm, e.g. -72 + frequency: int # centre frequency in MHz, e.g. 2437 + channel: int # channel number, e.g. 6 + signal_dbm: int # RSSI in dBm, e.g. -72 channel_width: int # MHz: 20 / 40 / 80 / 160; 0 if unknown - band: str # "2.4GHz" | "5GHz" | "6GHz" + band: str # "2.4GHz" | "5GHz" | "6GHz" class WirelessConfigDict(TypedDict): @@ -226,7 +232,7 @@ class Dot1XConfigDict(TypedDict): enabled: bool ssid: str auth_server: RadiusServerDict - acct_server: Optional[RadiusServerDict] + acct_server: RadiusServerDict | None reauth_interval: int pmksa_caching: bool @@ -263,7 +269,7 @@ class InterfaceConfigDict(TypedDict, total=False): mode: str access_vlan: int voice_vlan: int - trunk_vlans: List[int] + trunk_vlans: list[int] native_vlan: int @@ -280,7 +286,7 @@ class VlanConfigDict(TypedDict, total=False): name: str active: bool - interfaces: List[str] + interfaces: list[str] class STPInterfaceDict(TypedDict): @@ -297,11 +303,11 @@ class SpanningTreeDict(TypedDict): root_priority: int bridge_id: str bridge_priority: int - interfaces: Dict[str, STPInterfaceDict] + interfaces: dict[str, STPInterfaceDict] class PortChannelDict(TypedDict): - members: List[str] + members: list[str] protocol: str min_links: int is_up: bool @@ -314,7 +320,7 @@ class Dot1XPortDict(TypedDict): port_control: str host_mode: str auth_server: RadiusServerDict - acct_server: Optional[RadiusServerDict] + acct_server: RadiusServerDict | None reauthentication: bool reauth_interval: int guest_vlan: int @@ -334,7 +340,7 @@ class PoEPortDict(TypedDict): class PoESummaryDict(TypedDict): total_power_budget: float total_power_draw: float - ports: Dict[str, PoEPortDict] + ports: dict[str, PoEPortDict] # --------------------------------------------------------------------------- @@ -363,10 +369,10 @@ class KernelFactsDict(TypedDict): """ release: str # uname -r - loaded: Optional[List[str]] # /proc/modules - builtin: Optional[List[str]] # modules.builtin -- compiled into the kernel image - available: Optional[List[str]] # modules.dep -- shipped as loadable modules - config: Optional[Dict[str, str]] # build configuration, set options only; quotes stripped + loaded: list[str] | None # /proc/modules + builtin: list[str] | None # modules.builtin -- compiled into the kernel image + available: list[str] | None # modules.dep -- shipped as loadable modules + config: dict[str, str] | None # build configuration, set options only; quotes stripped class ListeningSocketDict(TypedDict): @@ -381,11 +387,11 @@ class ListeningSocketDict(TypedDict): proto: str # "tcp" or "udp" address: str # "0.0.0.0", "::", "*", "127.0.0.1", "::ffff:127.0.0.1", ... port: int - interface: Optional[str] # the %zone a socket is bound to ("lo", "eth0"), if any - process: Optional[str] # the first process holding it; None without one or without root - pid: Optional[int] - unit: Optional[str] # the process's systemd service, without ".service" - container_id: Optional[str] # the full container ID, for a container on the host network + interface: str | None # the %zone a socket is bound to ("lo", "eth0"), if any + process: str | None # the first process holding it; None without one or without root + pid: int | None + unit: str | None # the process's systemd service, without ".service" + container_id: str | None # the full container ID, for a container on the host network class ListeningSocketsDict(TypedDict): @@ -397,36 +403,167 @@ class ListeningSocketsDict(TypedDict): """ attributed: bool - sockets: List[ListeningSocketDict] + sockets: list[ListeningSocketDict] + + +class NetfilterMatchDict(TypedDict): + """One condition of a netfilter rule. + + ``field`` is one of ``iifname``, ``oifname``, ``saddr``, ``daddr``, + ``l4proto``, ``dport``, ``sport``, ``ct_state``, ``ct_orig_dport``, + ``nfproto`` -- or ``unknown``, whose single value is the condition's text. + The rule matches when the field's value is (``==``) or is not (``!=``) one + of ``values``: interface names (``eth*`` a wildcard), CIDRs, protocol names, + ports (``8000-8010`` a range), conntrack states in lower case. + """ + + field: str + op: str # "==" or "!=" + values: list[str] + + +class NetfilterRuleDict(TypedDict): + """One rule: every match must hold, then the verdict applies. + + ``verdict`` is ``accept``, ``drop``, ``reject``, ``return``, ``jump`` or + ``goto`` (to ``target``), ``continue`` (a rule that only counts, logs or + marks), or ``other`` -- NAT, a queue, a verdict map -- with ``target`` naming it. + """ + + matches: list[NetfilterMatchDict] + verdict: str + target: str | None + text: str # the rule as iptables-save printed it, or as one nft-style line + + +class NetfilterChainDict(TypedDict): + """A chain. A base chain has a ``hook``, ``priority`` and ``policy``; a chain + reached only by ``jump``/``goto`` has None for all three.""" + + family: str # "ip", "ip6", "inet", "bridge", "arp", "netdev" + table: str + name: str + type: str | None # "filter", "nat", "route" + hook: str | None # "prerouting", "input", "forward", "output", "postrouting", ... + priority: int | None + policy: str | None # "accept" or "drop" + rules: list[NetfilterRuleDict] + + +class HostFirewallDict(TypedDict): + """What ``HostFirewallMixin.get_host_firewall`` read. + + ``sources`` names what was read: ``nftables``, ``iptables-nft`` (the tables + iptables manages, taken from ``iptables-save`` because nft shows their + extensions opaquely) and ``iptables-legacy`` (a ruleset of its own). + """ + + sources: list[str] + chains: list[NetfilterChainDict] class VirtualIpDict(TypedDict): """An address the host's HA configuration lets float between machines. Declared, not observed: whether the host holds the address right now is in - its interface addresses, not here. Only these fields leave the host -- never - a password, an etcd endpoint or a notify script. + its interface addresses, not here -- unless the mechanism says so itself. + CARP does (``holds``): a firewall's interface list need not show the + address at all, but CARP's state names the MASTER. Only these fields leave + the host -- never a password, an etcd endpoint or a notify script. """ - mechanism: str # "vip-manager" | "keepalived" - instance: str # the vrrp_instance name, or the vip-manager unit without ".service" + mechanism: str # "vip-manager" | "keepalived" | "carp" + #: the vrrp_instance name, the vip-manager unit without ".service", or the + #: CARP address's interface description + instance: str address: str # normalised: "10.7.224.10", "fd00::10" - prefix_length: Optional[int] # None where the configuration gives none - interface: Optional[str] - group_key: Optional[str] # vip-manager's trigger-key, or "vrid:" for keepalived - running: Optional[bool] # the unit is active + prefix_length: int | None # None where the configuration gives none + interface: str | None + #: vip-manager's trigger-key, "vrid:" for keepalived, "vhid:" for CARP + group_key: str | None + running: bool | None # the unit is active; for CARP: in MASTER or BACKUP + #: The host holds the address now, by the mechanism's own state (CARP's + #: MASTER). Absent or None: the interface addresses tell. + holds: NotRequired[bool | None] + + +class ServiceClusterMemberDict(TypedDict): + """One member of a service cluster, as the configuration names it. + + A member is named by what the configuration has: an address (a URL's host, a + whitelisted address -- possibly a host name), a port where it gives one, a + name where the mechanism has its own (Patroni's). ``this_host`` marks the + member the reading host is when the configuration does not name it by + address. Only these fields leave the host -- never a password or a DCS + endpoint. + """ + + name: str | None + address: str | None + port: int | None + #: as the mechanism words it: Patroni's "leader"/"sync_standby"/"replica", + #: adguardhome-sync's "origin"/"replica", BunkerWeb's "manager"/"worker" + role: str | None + #: the member's state where the mechanism reports one (Patroni's "running", + #: "streaming"); configuration alone has none + state: str | None + this_host: bool + #: the index in ``members`` of the member whose configuration this one takes: + #: a replica's origin, a worker's manager. None where it takes none. + config_from: int | None + + +class ServiceClusterDict(TypedDict): + """A cluster the host's configuration says it belongs to (netOrk #1125). + + The evidence that two hosts run one service together, not that they merely + answer at one address: Patroni's scope and the members its DCS lists, the + origin and replicas adguardhome-sync keeps in step, a BunkerWeb worker's + manager. + """ + + mechanism: str # "patroni" | "adguardhome-sync" | "bunkerweb" + #: what the members run, as a host reports it: "patroni", "adguardhome", "bunkerweb" + service: str + #: what the evidence was read from on this host: the unit without ".service" + #: ("patroni", "patroni@16-main") or the container's name ("adguardhome-sync") + source: str + #: what the members share where the reading host can name it: Patroni's scope, + #: the sync's origin ("10.9.100.5:3000"), a worker's manager address. None for a + #: manager, which cannot name its own address. + cluster: str | None + running: bool | None # the unit is active; a listed container runs + members: list[ServiceClusterMemberDict] + + +class ServiceClustersDict(TypedDict): + """What a driver's ``get_service_clusters`` read, per mechanism. + + A list is what was looked at -- ``[]`` a host without that mechanism. None is + "did not look": a configuration it could not read, a host without systemd, a + container engine it may not ask, a sync configured by file. + """ + + patroni: list[ServiceClusterDict] | None + adguardhome_sync: list[ServiceClusterDict] | None + bunkerweb: list[ServiceClusterDict] | None class VirtualIpsDict(TypedDict): - """What ``VirtualIpsMixin.get_virtual_ips`` read, per mechanism. + """What a driver's ``get_virtual_ips`` read, per mechanism. A list is what was looked at -- ``[]`` a host without that mechanism. None is "did not look": a configuration it could not read, an include it could not - follow, a host without systemd. A consumer keeps what it knew for a None. + follow, a host without systemd, a mechanism the driver does not read (a + firewall's reading has ``vip_manager`` and ``keepalived`` None). A consumer + keeps what it knew for a None, and for a missing ``carp``. """ - vip_manager: Optional[List[VirtualIpDict]] - keepalived: Optional[List[VirtualIpDict]] + vip_manager: list[VirtualIpDict] | None + keepalived: list[VirtualIpDict] | None + #: A firewall's CARP addresses (not read by ``VirtualIpsMixin``, which reads Linux + #: hosts): every address bound to a VHID, the CARP address and IP aliases on it. + carp: NotRequired[list[VirtualIpDict] | None] class PortForwardDict(TypedDict): @@ -445,7 +582,7 @@ class PortForwardDict(TypedDict): class SecurityZoneDict(TypedDict): - interfaces: List[str] + interfaces: list[str] policy: str description: str @@ -491,12 +628,12 @@ class FirewallRuleDict(TypedDict): class FirewallRuleUpdateDict(TypedDict): uuid: str rule: FirewallRuleDict - changed_fields: List[str] + changed_fields: list[str] class FirewallRuleDiffDict(TypedDict): - add: List[FirewallRuleDict] - update: List[FirewallRuleUpdateDict] + add: list[FirewallRuleDict] + update: list[FirewallRuleUpdateDict] class DhcpReservationDict(TypedDict): @@ -525,12 +662,12 @@ class DhcpReservationDict(TypedDict): class DhcpReservationUpdateDict(TypedDict): uuid: str reservation: DhcpReservationDict - changed_fields: List[str] + changed_fields: list[str] class DhcpReservationDiffDict(TypedDict): - add: List[DhcpReservationDict] - update: List[DhcpReservationUpdateDict] + add: list[DhcpReservationDict] + update: list[DhcpReservationUpdateDict] class DhcpOptionDataDict(TypedDict, total=False): @@ -545,11 +682,11 @@ class DhcpOptionDataDict(TypedDict, total=False): desired value is empty". A driver must preserve options it was not given. """ - routers: List[str] - domain_name_servers: List[str] + routers: list[str] + domain_name_servers: list[str] domain_name: str - domain_search: List[str] - ntp_servers: List[str] + domain_search: list[str] + ntp_servers: list[str] class DhcpSubnetDict(TypedDict): @@ -571,7 +708,7 @@ class DhcpSubnetDict(TypedDict): uuid: str subnet: str description: str - pools: List[str] + pools: list[str] option_data: DhcpOptionDataDict match_client_id: bool @@ -579,12 +716,54 @@ class DhcpSubnetDict(TypedDict): class DhcpSubnetUpdateDict(TypedDict): uuid: str subnet: DhcpSubnetDict - changed_fields: List[str] + changed_fields: list[str] class DhcpSubnetDiffDict(TypedDict): - add: List[DhcpSubnetDict] - update: List[DhcpSubnetUpdateDict] + add: list[DhcpSubnetDict] + update: list[DhcpSubnetUpdateDict] + + +class DhcpHaPeerDict(TypedDict): + """One server of a DHCP HA relationship, as the server's configuration names it.""" + + name: str + #: The peer's role as the server words it ("primary", "standby", ...) -- data, + #: not a vocabulary every server shares. + role: str + #: Where the peer's HA listener answers, when the server addresses it by URL. + url: NotRequired[str] + + +class DhcpHaDict(TypedDict): + """What a DHCP server's own HA configuration declares. + + Configured, not observed: whether the pair is in hot standby right now, or + one side has taken over, is the server's state and not in here. + """ + + #: The name of the service this configuration belongs to, as ``get_services`` + #: reports it -- the caller matches it to that service. + service: NotRequired[str] + enabled: bool + #: Which of ``peers`` is this server. + this_server: NotRequired[str] + peers: list[DhcpHaPeerDict] + + +class ConfigSyncDict(TypedDict): + """Where a host pushes its configuration, and whose. + + ``peer`` None: the host pushes to no one (the receiving side of a pair, or a + host without the feature switched on). + """ + + peer: str | None + #: The services whose configuration it pushes, by the name ``get_services`` + #: reports. + services: list[str] + #: Sync items the driver has no service name for, as the host names them. + unmapped: list[str] class VPNTunnelDict(TypedDict): @@ -661,8 +840,8 @@ class VMDict(TypedDict): class VMPassthroughDict(TypedDict): """A host device handed through to a VM (PCI, USB).""" - slot: str # hypervisor's device key, e.g. "hostpci0" - kind: str # "pci" or "usb" + slot: str # hypervisor's device key, e.g. "hostpci0" + kind: str # "pci" or "usb" config: str # hypervisor's own description of the device @@ -672,19 +851,19 @@ class VMConfigDict(TypedDict): vcpus: int memory: int os_type: str - boot_order: List[str] - disks: List[VMDiskDict] - nics: List[VMNICDict] + boot_order: list[str] + disks: list[VMDiskDict] + nics: list[VMNICDict] description: str - tags: List[str] + tags: list[str] # Hardware details not every hypervisor exposes; absent when unknown. - os_name: NotRequired[str] # human-readable guest OS, e.g. "Ubuntu Linux (64-bit)" - cpu_type: NotRequired[str] # e.g. "host", "kvm64" + os_name: NotRequired[str] # human-readable guest OS, e.g. "Ubuntu Linux (64-bit)" + cpu_type: NotRequired[str] # e.g. "host", "kvm64" sockets: NotRequired[int] cores_per_socket: NotRequired[int] - firmware: NotRequired[str] # "bios" or "efi" - machine: NotRequired[str] # machine type / virtual hardware version - passthrough: NotRequired[List[VMPassthroughDict]] + firmware: NotRequired[str] # "bios" or "efi" + machine: NotRequired[str] # machine type / virtual hardware version + passthrough: NotRequired[list[VMPassthroughDict]] class StorageVolumeDict(TypedDict): @@ -727,29 +906,29 @@ class PhysicalDiskDict(TypedDict): model: str serial: str vendor: str - type: str # "hdd", "ssd", "nvme" - size: int # bytes - rpm: int # 0 for SSD/NVMe - temperature: int # Celsius; -1 if unavailable - health: str # "healthy", "warning", "failed", "unknown" - pool: str # name of the containing pool; empty string if spare/unassigned + type: str # "hdd", "ssd", "nvme" + size: int # bytes + rpm: int # 0 for SSD/NVMe + temperature: int # Celsius; -1 if unavailable + health: str # "healthy", "warning", "failed", "unknown" + pool: str # name of the containing pool; empty string if spare/unassigned class DiskPoolDict(TypedDict): """A RAID array, ZFS pool, or volume group.""" name: str - type: str # "zfs", "lvm", "md", "hardware-raid", "btrfs" - level: str # "stripe", "mirror", "raidz1", "raidz2", "raidz3", - # "raid0" … "raid60", "single", etc. - status: str # "online", "degraded", "faulted", "offline", "unknown" - total: int # bytes - used: int # bytes - available: int # bytes - disks: List[str] # slot identifiers of member disks + type: str # "zfs", "lvm", "md", "hardware-raid", "btrfs" + level: str # "stripe", "mirror", "raidz1", "raidz2", "raidz3", + # "raid0" … "raid60", "single", etc. + status: str # "online", "degraded", "faulted", "offline", "unknown" + total: int # bytes + used: int # bytes + available: int # bytes + disks: list[str] # slot identifiers of member disks auto_expand: bool dedup: bool - compression: str # "off", "lz4", "gzip", "zstd", etc. + compression: str # "off", "lz4", "gzip", "zstd", etc. class LogicalVolumeDict(TypedDict): @@ -757,74 +936,74 @@ class LogicalVolumeDict(TypedDict): name: str pool: str - type: str # "filesystem", "volume" (block device / LUN), "zvol" - total: int # bytes - used: int # bytes - available: int # bytes - mountpoint: str # empty string for block volumes - compression: str # "off", "lz4", etc. + type: str # "filesystem", "volume" (block device / LUN), "zvol" + total: int # bytes + used: int # bytes + available: int # bytes + mountpoint: str # empty string for block volumes + compression: str # "off", "lz4", etc. dedup: bool readonly: bool - snapshots: int # number of snapshots currently held + snapshots: int # number of snapshots currently held class NASShareDict(TypedDict): """A network share or iSCSI target exported by the storage device.""" name: str - protocol: str # "nfs", "smb", "afp", "ftp", "sftp", "iscsi", "webdav" - path: str # local filesystem path or iSCSI target IQN - volume: str # logical volume or dataset this share is backed by + protocol: str # "nfs", "smb", "afp", "ftp", "sftp", "iscsi", "webdav" + path: str # local filesystem path or iSCSI target IQN + volume: str # logical volume or dataset this share is backed by enabled: bool readonly: bool description: str - clients: List[str] # IP/subnet allow-list; empty list = all hosts allowed + clients: list[str] # IP/subnet allow-list; empty list = all hosts allowed class VolumeSnapshotDict(TypedDict): """A point-in-time snapshot of a logical volume or dataset.""" name: str - volume: str # parent volume / dataset - created: float # Unix epoch - size: int # bytes of unique data referenced by this snapshot + volume: str # parent volume / dataset + created: float # Unix epoch + size: int # bytes of unique data referenced by this snapshot description: str - clones: List[str] # volumes cloned from this snapshot + clones: list[str] # volumes cloned from this snapshot class StorageQuotaDict(TypedDict): """A filesystem quota applied to a user, group, or dataset.""" - target: str # username, group name, or dataset path - target_type: str # "user", "group", "dataset" - volume: str # volume / dataset the quota applies to - used: int # bytes currently used - quota: int # hard limit in bytes; 0 = no limit - ref_quota: int # referenced-data limit in bytes; 0 = no limit + target: str # username, group name, or dataset path + target_type: str # "user", "group", "dataset" + volume: str # volume / dataset the quota applies to + used: int # bytes currently used + quota: int # hard limit in bytes; 0 = no limit + ref_quota: int # referenced-data limit in bytes; 0 = no limit class StorageServiceDict(TypedDict): """Status of a file-sharing or access service running on the device.""" - name: str # "nfs", "smb", "ftp", "ssh", "iscsi", "webdav", etc. - enabled: bool # administratively enabled - running: bool # currently active / listening - port: int # primary listening port; 0 if N/A - version: str # protocol version string (e.g. "4.1" for NFSv4.1) + name: str # "nfs", "smb", "ftp", "ssh", "iscsi", "webdav", etc. + enabled: bool # administratively enabled + running: bool # currently active / listening + port: int # primary listening port; 0 if N/A + version: str # protocol version string (e.g. "4.1" for NFSv4.1) class ReplicationJobDict(TypedDict): """A scheduled replication or sync task.""" name: str - source: str # source path / dataset - target: str # destination path / dataset (may be remote: host:path) - direction: str # "push" or "pull" - schedule: str # cron expression or human label ("daily", "hourly") + source: str # source path / dataset + target: str # destination path / dataset (may be remote: host:path) + direction: str # "push" or "pull" + schedule: str # cron expression or human label ("daily", "hourly") enabled: bool - last_run: float # Unix epoch; 0.0 if never run - last_status: str # "success", "failed", "running", "pending" - bytes_sent: int # bytes transferred in the last run + last_run: float # Unix epoch; 0.0 if never run + last_status: str # "success", "failed", "running", "pending" + bytes_sent: int # bytes transferred in the last run # --------------------------------------------------------------------------- @@ -840,7 +1019,7 @@ class UserDict(TypedDict): gid: int home: str shell: str - groups: List[str] # all supplementary group names + groups: list[str] # all supplementary group names class ProcessDict(TypedDict): @@ -849,30 +1028,30 @@ class ProcessDict(TypedDict): pid: int ppid: int user: str - cpu: float # CPU utilisation 0.0–100.0 - memory: float # RSS percentage of total RAM 0.0–100.0 - vsz: int # virtual memory size in KiB - rss: int # resident set size in KiB - tty: str # controlling terminal; empty string if none - state: str # "R" running, "S" sleeping, "D" uninterruptible, "Z" zombie, etc. - started: str # start time string as reported by ``ps`` (e.g. "12:34" or "May28") - command: str # full command line + cpu: float # CPU utilisation 0.0–100.0 + memory: float # RSS percentage of total RAM 0.0–100.0 + vsz: int # virtual memory size in KiB + rss: int # resident set size in KiB + tty: str # controlling terminal; empty string if none + state: str # "R" running, "S" sleeping, "D" uninterruptible, "Z" zombie, etc. + started: str # start time string as reported by ``ps`` (e.g. "12:34" or "May28") + command: str # full command line class CronJobDict(TypedDict): """A scheduled cron task.""" - user: str # owner of the crontab entry - schedule: str # five-field cron expression, e.g. "0 * * * *" - command: str # shell command to execute + user: str # owner of the crontab entry + schedule: str # five-field cron expression, e.g. "0 * * * *" + command: str # shell command to execute description: NotRequired[str] # optional inline comment class ApplyUpdatesResultDict(TypedDict): """Result of an ``apply_updates()`` call.""" - success: bool # True if the package manager exited without error - output: str # raw stdout/stderr captured from the package manager + success: bool # True if the package manager exited without error + output: str # raw stdout/stderr captured from the package manager error: NotRequired[str] # error message if success is False @@ -883,24 +1062,24 @@ class ContainerEngineDict(TypedDict): it is read and modelled above the driver, in netOrk (netork#765). """ - engine: str # "docker" - api: str # "docker-engine": the Docker Engine API over ``system dial-stdio`` + engine: str # "docker" + api: str # "docker-engine": the Docker Engine API over ``system dial-stdio`` class DeviceActionResultDict(TypedDict): """Return value of ``run_device_action()``.""" - success: bool # True if the action completed without error - output: str # human-readable output or status message + success: bool # True if the action completed without error + output: str # human-readable output or status message class SNMPConfigDict(TypedDict): """SNMP agent configuration detected on the device.""" - running: bool # True if the SNMP daemon is active - community: str # read community string (e.g. "public") - port: int # listening port (default 161) - version: str # highest supported version: "1", "2c", or "3" + running: bool # True if the SNMP daemon is active + community: str # read community string (e.g. "public") + port: int # listening port (default 161) + version: str # highest supported version: "1", "2c", or "3" # --------------------------------------------------------------------------- @@ -916,7 +1095,7 @@ class HealthMetricsIfaceDict(TypedDict): tx_bytes: NotRequired[int] rx_errors: NotRequired[int] tx_errors: NotRequired[int] - tx_drops: NotRequired[int] # populated instead of tx_errors when the driver reports drops + tx_drops: NotRequired[int] # populated instead of tx_errors when the driver reports drops speed_mbps: NotRequired[int] @@ -935,7 +1114,126 @@ class HealthMetricsDict(TypedDict): load_1: NotRequired[float] load_5: NotRequired[float] load_15: NotRequired[float] - interfaces: NotRequired[Dict[str, HealthMetricsIfaceDict]] + interfaces: NotRequired[dict[str, HealthMetricsIfaceDict]] + + +LicenseStatus = Literal["active", "missing", "expired", "invalid", "none", "unknown"] + + +class LicenseDict(TypedDict): + """A license or subscription the device reports (``LicenseMixin.get_licenses``). + + A fact, not a judgement: whether a free edition or a missing subscription is + a problem is the caller's to decide. + """ + + name: str # what it licenses, as the device names it + status: LicenseStatus # "none": the device runs without one, as installed + edition: NotRequired[str] # the edition or level, e.g. "Free", "Enterprise Plus" + expires: NotRequired[str | None] # ISO date, None for no expiry + detail: NotRequired[str] # the device's own words, e.g. why a feature is refused + # What the device withholds without it while the rest works, e.g. ["drives", + # "firmware inventory"] on a BMC that answers but wants a further license. + # Absent: the device says nothing of the kind, or withholds everything it + # would offer with it. + withholds: NotRequired[list[str]] + + +UpdateState = Literal["idle", "running", "done", "failed"] + + +class UpdateProgressDict(TypedDict): + """Where an update the device runs by itself stands + (``UpdateMixin.get_update_progress``).""" + + state: UpdateState # "idle": none has run since the device started, or none is known + log: str # the device's own log of it, whole; a caller follows it by line count + + +class DnsForwardDict(TypedDict): + """A domain the device's resolver forwards to another server + (``DnsForwardMixin.get_dns_forwards``).""" + + domain: str # without the trailing dot + server: str + port: int + enabled: bool + + +class OspfInterfaceDict(TypedDict): + """OSPF on one interface (``OspfMixin.get_ospf``).""" + + address: str + prefix_length: int + area: str + network_type: str # as the routing daemon names it, e.g. "POINTOPOINT" + authentication: str # as the routing daemon names it, e.g. "MessageDigest"; "" for none + neighbors: int + neighbors_adjacent: int # neighbours with a full adjacency + cost: int + + +class OspfDict(TypedDict): + """The device's OSPF state (``OspfMixin.get_ospf``).""" + + router_id: str | None + areas: list[str] + interfaces: dict[str, OspfInterfaceDict] # by interface name + + +class FirmwareOfferDict(TypedDict): + """The firmware a device runs and the newest its vendor offers + (``FirmwareOfferMixin.get_firmware_offer``). A fact, not a judgement.""" + + current_version: str + latest_version: str + newer: bool # the offer is newer than what runs, by the vendor's versioning + download_url: NotRequired[str] + + +class HypervisorHostDict(TypedDict): + """One host of a hypervisor or cluster (``HypervisorDriver.get_hypervisor_hosts``).""" + + name: str + connected: bool | None # reachable by its manager; None where it cannot tell + maintenance: bool | None # in maintenance mode + config_issues: list[str] # the manager's configuration warnings, verbatim + managed_by: NotRequired[str | None] # the address of a central manager, if any + + +class FixStepOutcomeDict(TypedDict): + """What one step of a fix routine reports (see ``napalm_device_types.fixes``).""" + + ok: bool + output: str + + +class FixStepDict(TypedDict): + """One step of a fix routine, as ``FixResultDict`` lists it.""" + + step: str + ok: bool + output: str + + +class FixResultDict(TypedDict): + """The result of a fix routine: every step that ran, up to the first failure.""" + + success: bool + steps: list[FixStepDict] + output: str # the steps' output, one "[step] output" line each + + +class SnmpAgentSettingsDict(TypedDict): + """What the caller wants an SNMP agent to be (``SnmpAgentMixin``).""" + + community: str # the read-only v2c community + allowed_sources: list[ + str + ] # addresses or CIDRs let through to the agent; [] changes no firewall + contact: NotRequired[str] # sysContact + location: NotRequired[str] # sysLocation + listen: NotRequired[str] # the address the agent binds to; absent: every address class NICConfigDict(TypedDict): @@ -971,17 +1269,17 @@ class NetworkTargetDict(TypedDict): Distinguishes real bridges (Linux or OVS) from SDN network segments (vnets), and tells the caller whether a separate ``vlan_tag`` may be applied on top: - Linux bridge: vlan_aware reflects the bridge's own ``bridge_vlan_aware`` flag. - - OVS bridge: always vlan_aware (OVS bridges tag per-port regardless of a - dedicated "VLAN aware" setting). - - SDN vnet: never vlan_aware — the VLAN is already fixed by the vnet's zone/tag, - so a NIC attached to it must not also carry a ``vlan_tag``. That fixed VLAN - is surfaced via ``fixed_vlan_tag`` instead, for display purposes. + - OVS bridge: always vlan_aware (OVS tags per port, with no "VLAN aware" setting). + - SDN vnet: never vlan_aware — its zone/tag fixes the VLAN (``fixed_vlan_tag``). + ``uplinks``: the physical NICs it leaves the host through, one group per way out + (a bond's slaves together, a single NIC alone); no internal, VLAN or virtual ports. """ name: str # Bridge, vnet or port group name, usable directly as NICConfigDict.bridge kind: str # "bridge", "vnet" or "portgroup" (VMware: VLAN fixed like a vnet's) vlan_aware: bool # True if a NICConfigDict.vlan_tag may be set on top of this target fixed_vlan_tag: NotRequired[int | None] # vnet only: the VLAN ID already baked into it + uplinks: NotRequired[list[list[str]]] # NIC groups it leaves through; absent = unknown class StorageTargetDict(TypedDict): @@ -1009,7 +1307,7 @@ class VMCpuTypeDict(TypedDict): name: str # Model name, usable directly as create_vm_from_cloud_init(cpu_type=...) description: str # One line on what the model is for, for a picker - features: List[str] # cpuinfo flags the guest is guaranteed to see + features: list[str] # cpuinfo flags the guest is guaranteed to see available: bool # False when this node's CPU cannot run the model default: bool # The model create_vm_from_cloud_init uses when cpu_type is None @@ -1022,19 +1320,19 @@ class VMCpuTypeDict(TypedDict): class PingSweepEntryDict(TypedDict): """Outcome of a single ``ping`` inside a sweep (see ``PingSweepMixin``).""" - ip: str # destination that was probed - alive: bool # True if at least one probe was answered - rtt_ms: Optional[float] # average round-trip time in ms; None if unreachable + ip: str # destination that was probed + alive: bool # True if at least one probe was answered + rtt_ms: float | None # average round-trip time in ms; None if unreachable error: NotRequired[str] # driver/transport error for this destination class PingSweepResultDict(TypedDict): """Result of a ``ping_sweep()`` call.""" - entries: List[PingSweepEntryDict] # one entry per probed destination, in input order - scanned: int # destinations actually probed - alive_count: int # entries with alive=True - truncated: bool # True if targets were dropped at the sweep cap + entries: list[PingSweepEntryDict] # one entry per probed destination, in input order + scanned: int # destinations actually probed + alive_count: int # entries with alive=True + truncated: bool # True if targets were dropped at the sweep cap # --------------------------------------------------------------------------- @@ -1050,29 +1348,29 @@ PowerAction = Literal[ "ForceRestart", "PowerCycle", ] -POWER_ACTIONS: Tuple[str, ...] = get_args(PowerAction) +POWER_ACTIONS: tuple[str, ...] = get_args(PowerAction) class PowerStateDict(TypedDict): """The managed server's power, as its BMC reports it.""" - state: str # "On", "Off", "PoweringOn" or "PoweringOff" - actions: List[str] # the PowerAction values this BMC accepts now + state: str # "On", "Off", "PoweringOn" or "PoweringOff" + actions: list[str] # the PowerAction values this BMC accepts now class BmcCpuDict(TypedDict): - socket: str # "CPU1", "Proc 1" - model: str # "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz" - cores: Optional[int] - threads: Optional[int] - status: str # "OK", "Warning", "Critical", "Absent" + socket: str # "CPU1", "Proc 1" + model: str # "Intel(R) Xeon(R) Silver 4314 CPU @ 2.40GHz" + cores: int | None + threads: int | None + status: str # "OK", "Warning", "Critical", "Absent" class BmcDimmDict(TypedDict): - slot: str # "DIMMA1", "PROC 1 DIMM 3" - size_bytes: Optional[int] - type: str # "DDR4", "DDR5"; empty when unknown - speed_mts: Optional[int] # MT/s + slot: str # "DIMMA1", "PROC 1 DIMM 3" + size_bytes: int | None + type: str # "DDR4", "DDR5"; empty when unknown + speed_mts: int | None # MT/s part_number: str serial_number: str status: str @@ -1082,9 +1380,9 @@ class BmcDriveDict(TypedDict): slot: str model: str serial_number: str - size_bytes: Optional[int] - media: str # "HDD", "SSD"; empty when unknown - protocol: str # "SATA", "SAS", "NVMe"; empty when unknown + size_bytes: int | None + media: str # "HDD", "SSD"; empty when unknown + protocol: str # "SATA", "SAS", "NVMe"; empty when unknown status: str @@ -1092,17 +1390,17 @@ class BmcPsuDict(TypedDict): slot: str model: str serial_number: str - capacity_watts: Optional[int] + capacity_watts: int | None status: str class BmcNicDict(TypedDict): """A network port of the managed server, not of the BMC.""" - name: str # "NIC.Integrated.1-1", "Port 1" - mac_address: str # lowercase, colon-separated; empty when unknown - speed_mbps: Optional[int] - link_up: Optional[bool] + name: str # "NIC.Integrated.1-1", "Port 1" + mac_address: str # lowercase, colon-separated; empty when unknown + speed_mbps: int | None + link_up: bool | None class HardwareInventoryDict(TypedDict): @@ -1110,35 +1408,35 @@ class HardwareInventoryDict(TypedDict): manufacturer: str model: str - serial_number: str # the server's, not the BMC's - system_uuid: str # SMBIOS system UUID, lowercase - cpus: List[BmcCpuDict] - memory: List[BmcDimmDict] - drives: List[BmcDriveDict] - power_supplies: List[BmcPsuDict] - nics: List[BmcNicDict] + serial_number: str # the server's, not the BMC's + system_uuid: str # SMBIOS system UUID, lowercase + cpus: list[BmcCpuDict] + memory: list[BmcDimmDict] + drives: list[BmcDriveDict] + power_supplies: list[BmcPsuDict] + nics: list[BmcNicDict] class EventLogEntryDict(TypedDict): """One entry of the BMC's system event log (SEL / Lifecycle / IML).""" id: str - timestamp: Optional[float] # Unix time; None when the BMC gave none - severity: str # "OK", "Warning" or "Critical" + timestamp: float | None # Unix time; None when the BMC gave none + severity: str # "OK", "Warning" or "Critical" message: str class FirmwareDict(TypedDict): """One firmware component: the BMC's own, BIOS, a controller, a NIC.""" - name: str # "BMC", "BIOS", "PERC H755 Front" + name: str # "BMC", "BIOS", "PERC H755 Front" version: str - updateable: Optional[bool] + updateable: bool | None class ProtocolStateDict(TypedDict): enabled: bool - port: Optional[int] # None for a protocol without a port of its own + port: int | None # None for a protocol without a port of its own class ManagementProtocolsDict(TypedDict): @@ -1154,10 +1452,12 @@ class ManagementProtocolsDict(TypedDict): class SystemIdentityDict(TypedDict): - """A host's SMBIOS identity, what ties it to its BMC (:mod:`napalm_device_types.system_identity`). + """A host's SMBIOS identity, what ties it to its BMC. + + See :mod:`napalm_device_types.system_identity`. Each value is empty when unknown or a vendor placeholder.""" - system_uuid: str # lowercase + system_uuid: str # lowercase product_serial: str board_serial: str diff --git a/napalm_device_types/nat_vpn.py b/napalm_device_types/nat_vpn.py index 5d8328a..e85b171 100644 --- a/napalm_device_types/nat_vpn.py +++ b/napalm_device_types/nat_vpn.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Address translation and VPN tunnels. A home gateway does a subset of what a firewall does, and these readers are @@ -11,7 +10,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import NATTranslationDict, PortForwardDict, VPNTunnelDict @@ -19,7 +18,7 @@ from napalm_device_types.models import NATTranslationDict, PortForwardDict, VPNT class NatVpnMixin: if TYPE_CHECKING: - def get_nat_translations(self) -> List[NATTranslationDict]: + def get_nat_translations(self) -> list[NATTranslationDict]: """ Returns a list of active NAT translation entries. @@ -47,7 +46,7 @@ class NatVpnMixin: """ ... - def get_port_forwards(self) -> List[PortForwardDict]: + def get_port_forwards(self) -> list[PortForwardDict]: """ Returns the port forwards that let traffic in from the WAN. @@ -87,7 +86,7 @@ class NatVpnMixin: """ ... - def get_vpn_tunnels(self) -> Dict[str, VPNTunnelDict]: + def get_vpn_tunnels(self) -> dict[str, VPNTunnelDict]: """ Returns the status of VPN tunnels. diff --git a/napalm_device_types/os.py b/napalm_device_types/os.py index 41785d0..8c62e09 100644 --- a/napalm_device_types/os.py +++ b/napalm_device_types/os.py @@ -10,11 +10,9 @@ Usage:: ... """ -from typing import List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.updates import UpdateMixin -from napalm_device_types.services import ServiceControlMixin -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( CronJobDict, @@ -23,9 +21,14 @@ from napalm_device_types.models import ( SNMPConfigDict, UserDict, ) +from napalm_device_types.packages import PackageManagementMixin +from napalm_device_types.services import ServiceControlMixin +from napalm_device_types.updates import UpdateMixin -class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver): +class OSDriver( + UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthMetricsMixin, DeviceTypeDriver +): """ Abstract intermediate driver for general-purpose operating systems (e.g. Linux, BSD, macOS). @@ -44,26 +47,20 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Interfaces matching this pattern are excluded from health-metric collection. # Set True for drivers where the out-error counter actually reports drops (e.g. OPNsense). - # ------------------------------------------------------------------ # Package management # ------------------------------------------------------------------ if TYPE_CHECKING: - - - - # ------------------------------------------------------------------ # Service management # ------------------------------------------------------------------ - # ------------------------------------------------------------------ # Users # ------------------------------------------------------------------ - def get_os_users(self) -> List[UserDict]: + def get_os_users(self) -> list[UserDict]: """ Returns a list of local OS user accounts. @@ -95,7 +92,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Processes # ------------------------------------------------------------------ - def get_processes(self) -> List[ProcessDict]: + def get_processes(self) -> list[ProcessDict]: """ Returns a snapshot of currently running processes. @@ -138,7 +135,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # Cron jobs # ------------------------------------------------------------------ - def get_cron_jobs(self) -> List[CronJobDict]: + def get_cron_jobs(self) -> list[CronJobDict]: """ Returns scheduled cron tasks from all user crontabs and ``/etc/cron.d``. @@ -166,7 +163,7 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM # SNMP # ------------------------------------------------------------------ - def get_snmp_config(self) -> Optional[SNMPConfigDict]: + def get_snmp_config(self) -> SNMPConfigDict | None: """ Returns the SNMP agent configuration currently active on the device, or ``None`` if no SNMP daemon is running or detectable. @@ -201,6 +198,11 @@ class OSDriver(UpdateMixin, ServiceControlMixin, PackageManagementMixin, HealthM """ Executes a named administrative action on the device. + **Deprecated** (docs/STRUCTURE.md): the actions become fix templates + in :mod:`napalm_device_types.fixes`, whose values the caller supplies -- + ``configure_snmp_agent``, ``configure_package_proxy``, + ``grant_container_engine_access``, ``ensure_package``. Goes in 5.0. + This method is an extensibility point for driver-specific one-off operations that do not fit any other NAPALM API method. Each driver documents the action names it supports. diff --git a/napalm_device_types/ospf.py b/napalm_device_types/ospf.py new file mode 100644 index 0000000..72be557 --- /dev/null +++ b/napalm_device_types/ospf.py @@ -0,0 +1,30 @@ +"""A device's OSPF state, in neutral terms. + +The routing daemon's own output (FRR, Quagga, a vendor's CLI) is the driver's +to translate; whether an interface without neighbours or without +authentication is a problem is the caller's to decide. +Declared only; a driver whose device runs OSPF implements it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from napalm_device_types.models import OspfDict + + +class OspfMixin: + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def get_ospf(self) -> OspfDict: + """Router ID, areas and OSPF per interface. + + Example:: + + {"router_id": "10.255.0.1", "areas": ["0.0.0.0"], + "interfaces": {"wg0": {"address": "10.255.0.1", "prefix_length": 24, + "area": "0.0.0.0", "network_type": "POINTOPOINT", + "authentication": "MessageDigest", "neighbors": 1, + "neighbors_adjacent": 1, "cost": 10}}} + """ + ... diff --git a/napalm_device_types/package_updates.py b/napalm_device_types/package_updates.py index 55e3542..2d1b69f 100644 --- a/napalm_device_types/package_updates.py +++ b/napalm_device_types/package_updates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Pending package updates: which package, from where, and whether it is a security fix. A patch deadline -- "security updates within 14 days" -- needs to know which @@ -16,7 +15,6 @@ security -- netOrk's CVE matching is what catches those. from __future__ import annotations import re -from typing import Dict, List, Set from napalm_device_types.models import UpdateDict from napalm_device_types.terminal import strip_terminal_codes @@ -30,9 +28,12 @@ APT_UPGRADABLE_COMMAND = "{ LC_ALL=C apt list --upgradable 2>/dev/null; echo __A #: Read-only. Lists the packages that a pending security advisory covers. A #: failed dnf prints nothing, the same as no advisories, so its exit status is -#: printed inside the group, as apt's is. +#: printed inside the group, as apt's is. It reads the cached metadata only, as +#: apt reads its local index: an expired cache otherwise makes dnf download the +#: repositories' metadata first, which can take minutes (#76). DNF_SECURITY_COMMAND = ( - "{ LC_ALL=C dnf updateinfo list --security --quiet 2>/dev/null; echo __DNF_RC=$?; } | cat" + "{ LC_ALL=C dnf --cacheonly updateinfo list --security --quiet 2>/dev/null;" + " echo __DNF_RC=$?; } | cat" ) # openssl/noble-updates,noble-security 3.0.13-0ubuntu3.6 amd64 [upgradable from: 3.0.13-0ubuntu3.5] @@ -42,10 +43,10 @@ _APT_STATUS = re.compile(r"^__APT_RC=(\d+)\s*$", re.MULTILINE) _DNF_STATUS = re.compile(r"^__DNF_RC=(\d+)\s*$", re.MULTILINE) -def _joined_lines(output: str) -> List[str]: +def _joined_lines(output: str) -> list[str]: """Lines as apt printed them: a terminal wraps long ones, and the continuation starts with a space.""" - lines: List[str] = [] + lines: list[str] = [] for line in output.splitlines(): if line.startswith(" ") and lines: lines[-1] += line[1:].rstrip() @@ -67,7 +68,7 @@ def _listing(output: str, status: re.Pattern[str], what: str) -> str: return status.sub("", text) -def parse_apt_upgradable(output: str) -> List[UpdateDict]: +def parse_apt_upgradable(output: str) -> list[UpdateDict]: """Parse :data:`APT_UPGRADABLE_COMMAND`'s output, one entry per package. A package listed for several architectures (``libc6`` for amd64 and i386) @@ -75,7 +76,7 @@ def parse_apt_upgradable(output: str) -> List[UpdateDict]: :raises ValueError: when apt failed or its exit status never arrived. """ - by_name: Dict[str, UpdateDict] = {} + by_name: dict[str, UpdateDict] = {} for line in _joined_lines(_listing(output, _APT_STATUS, "apt list --upgradable")): match = _APT_LINE.match(line) if not match: @@ -103,7 +104,7 @@ def nevra_name(nevra: str) -> str: return without_arch.rsplit("-", 2)[0] -def parse_dnf_security(output: str) -> Set[str]: +def parse_dnf_security(output: str) -> set[str]: """The names of the packages a pending security advisory covers. Parses :data:`DNF_SECURITY_COMMAND`'s ``ADVISORY SEVERITY/Sec. NEVRA`` lines. @@ -111,7 +112,7 @@ def parse_dnf_security(output: str) -> Set[str]: :raises ValueError: when dnf failed or its exit status never arrived -- every update would otherwise read as not security. """ - names: Set[str] = set() + names: set[str] = set() for line in _listing(output, _DNF_STATUS, "dnf updateinfo list --security").splitlines(): parts = line.split() if len(parts) >= 3 and parts[1].endswith("/Sec."): diff --git a/napalm_device_types/packages.py b/napalm_device_types/packages.py index 4a8f552..f7c682a 100644 --- a/napalm_device_types/packages.py +++ b/napalm_device_types/packages.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Listing and managing installed software. Identical on every device type that has a package manager -- this was five @@ -12,7 +11,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import PackageDict @@ -20,7 +19,7 @@ from napalm_device_types.models import PackageDict class PackageManagementMixin: if TYPE_CHECKING: - def get_packages(self) -> List[PackageDict]: + def get_packages(self) -> list[PackageDict]: """ Returns all packages currently known to the device's package manager (e.g. ``opkg`` on OpenWrt, ``apk`` on Alpine-based APs). @@ -97,7 +96,7 @@ class PackageManagementMixin: """ ... - def get_package_config(self, name: str) -> Dict[str, Any]: + def get_package_config(self, name: str) -> dict[str, Any]: """ Returns the current configuration of an installed package as a dictionary. The structure is package-specific. @@ -123,7 +122,7 @@ class PackageManagementMixin: """ ... - def set_package_config(self, name: str, config: Dict[str, Any]) -> None: + def set_package_config(self, name: str, config: dict[str, Any]) -> None: """ Writes a new configuration for an installed package. diff --git a/napalm_device_types/phone.py b/napalm_device_types/phone.py index 871a7cd..5409a60 100644 --- a/napalm_device_types/phone.py +++ b/napalm_device_types/phone.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Abstract base class for IP telephony endpoints. @@ -10,7 +9,7 @@ Usage:: ... """ -from typing import TYPE_CHECKING, Any, Dict, List +from typing import TYPE_CHECKING, Any from napalm_device_types.base import DeviceTypeDriver @@ -36,7 +35,7 @@ class PhoneDriver(DeviceTypeDriver): if TYPE_CHECKING: - def get_sip_accounts(self) -> List[Dict[str, Any]]: + def get_sip_accounts(self) -> list[dict[str, Any]]: """ Returns the SIP registrations configured on the phone. @@ -60,7 +59,7 @@ class PhoneDriver(DeviceTypeDriver): """ ... - def get_call_status(self) -> Dict[str, Any]: + def get_call_status(self) -> dict[str, Any]: """ Returns what the phone is doing right now. diff --git a/napalm_device_types/ping_sweep.py b/napalm_device_types/ping_sweep.py index ddd92c0..1caa9b2 100644 --- a/napalm_device_types/ping_sweep.py +++ b/napalm_device_types/ping_sweep.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Generic ICMP sweep on top of the NAPALM-standard ``ping()``. Sweeping a range of addresses is orchestration, not device mechanics: the @@ -21,7 +20,8 @@ target list (see ``PING_SWEEP_MAX_TARGETS``). from __future__ import annotations -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterable, List, Optional +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any, ClassVar from napalm.base import NetworkDriver @@ -60,7 +60,7 @@ class PingSweepMixin: #: ``False`` opts a driver out of ping sweeps even though it implements #: ``ping()``. ``None`` (the default) means "decide by introspection". - SUPPORTS_PING: ClassVar[Optional[bool]] = None + SUPPORTS_PING: ClassVar[bool | None] = None if TYPE_CHECKING: # pragma: no cover - declared for type checkers only @@ -73,7 +73,7 @@ class PingSweepMixin: size: int = 100, count: int = 5, vrf: str = "", - ) -> Dict[str, Any]: ... + ) -> dict[str, Any]: ... @classmethod def supports_ping(cls) -> bool: @@ -86,9 +86,9 @@ class PingSweepMixin: *, count: int = 1, timeout: int = 1, - max_targets: Optional[int] = None, - on_progress: Optional[Callable[[int, int], None]] = None, - should_stop: Optional[Callable[[], bool]] = None, + max_targets: int | None = None, + on_progress: Callable[[int, int], None] | None = None, + should_stop: Callable[[], bool] | None = None, ) -> PingSweepResultDict: """Ping every address in *destinations* and report who answered. @@ -115,7 +115,7 @@ class PingSweepMixin: targets = targets[:limit] total = len(targets) - entries: List[PingSweepEntryDict] = [] + entries: list[PingSweepEntryDict] = [] for done, destination in enumerate(targets, start=1): if should_stop is not None and should_stop(): break @@ -151,7 +151,12 @@ class PingSweepMixin: success = reply.get("success") or {} if not isinstance(success, dict): - return {"ip": destination, "alive": False, "rtt_ms": None, "error": "malformed ping reply"} + return { + "ip": destination, + "alive": False, + "rtt_ms": None, + "error": "malformed ping reply", + } probes_sent = _as_int(success.get("probes_sent")) packet_loss = _as_int(success.get("packet_loss"), default=probes_sent) alive = bool(success.get("results")) or probes_sent > packet_loss @@ -167,7 +172,7 @@ def _as_int(value: Any, default: int = 0) -> int: return default -def _as_float(value: Any) -> Optional[float]: +def _as_float(value: Any) -> float | None: try: return float(value) except (TypeError, ValueError): diff --git a/napalm_device_types/provisioning.py b/napalm_device_types/provisioning.py index 8879934..b479614 100644 --- a/napalm_device_types/provisioning.py +++ b/napalm_device_types/provisioning.py @@ -3,8 +3,9 @@ What provisioning a VM from a cloud image needs to know about the guest. The same for every hypervisor: which agent a guest runs so a QEMU-based hypervisor can read its IP, the network-config cloud-init and FreeBSD's -nuageinit both read, and how a packed cloud image is unpacked. How a -hypervisor attaches, boots and talks to the VM stays in its driver. +nuageinit both read, how a packed cloud image is unpacked, and how a failure +names the VM it left behind. How a hypervisor attaches, boots and talks to +the VM stays in its driver. """ from __future__ import annotations @@ -37,6 +38,30 @@ _DECOMPRESSORS = { } +class VMProvisionIncomplete(RuntimeError): + """``create_vm_from_cloud_init`` failed after the VM was created. + + The VM is still on the hypervisor, half built: ``vmid`` and ``node`` name it + as ``VMProvisionResultDict`` would have, so the caller can remove it with + ``destroy_vm``. The failure itself is the ``__cause__`` (``raise ... from``), + and the message is its own. A failure before the VM exists raises as before. + """ + + def __init__(self, message: str, *, vmid: str, node: str) -> None: + super().__init__(message) + self.vmid = vmid + self.node = node + + def __reduce__(self) -> tuple[Any, ...]: + # The keyword-only arguments are not in ``args``, which is all that + # pickle passes back by default. + return (_vm_provision_incomplete, (str(self), self.vmid, self.node)) + + +def _vm_provision_incomplete(message: str, vmid: str, node: str) -> VMProvisionIncomplete: + return VMProvisionIncomplete(message, vmid=vmid, node=node) + + def network_config(nics: list[tuple[str, bool]]) -> dict[str, Any] | None: """Network-config v2: DHCP on every ``(mac, dhcp)`` NIC that asks for it. diff --git a/napalm_device_types/residential_gateway.py b/napalm_device_types/residential_gateway.py index 369bb7a..b2c9567 100644 --- a/napalm_device_types/residential_gateway.py +++ b/napalm_device_types/residential_gateway.py @@ -19,11 +19,11 @@ Usage:: ... """ -from typing import Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.nat_vpn import NatVpnMixin -from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.dhcp import DhcpServerMixin +from napalm_device_types.health_metrics import HealthMetricsMixin from napalm_device_types.models import ( HostDict, RadioStatusDict, @@ -31,6 +31,7 @@ from napalm_device_types.models import ( WANStatusDict, WirelessClientDict, ) +from napalm_device_types.nat_vpn import NatVpnMixin class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, DeviceTypeDriver): @@ -48,8 +49,6 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, ROLE: str = "residential_gateway" TYPE_LABEL: str = "Gateway" - - if TYPE_CHECKING: def get_wan_status(self) -> WANStatusDict: @@ -85,7 +84,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_hosts(self) -> List[HostDict]: + def get_hosts(self) -> list[HostDict]: """ Returns the list of hosts known to the gateway (LAN clients). @@ -113,9 +112,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - - - def get_wireless_clients(self) -> List[WirelessClientDict]: + def get_wireless_clients(self) -> list[WirelessClientDict]: """ Returns the list of wireless clients currently associated with the device's built-in access point(s). @@ -125,7 +122,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_ssids(self) -> Dict[str, SSIDDict]: + def get_ssids(self) -> dict[str, SSIDDict]: """ Returns the configured wireless networks (SSIDs). @@ -134,7 +131,7 @@ class ResidentialGatewayDriver(NatVpnMixin, HealthMetricsMixin, DhcpServerMixin, """ ... - def get_radio_status(self) -> Dict[str, RadioStatusDict]: + def get_radio_status(self) -> dict[str, RadioStatusDict]: """ Returns the status of the device's wireless radios. diff --git a/napalm_device_types/roles.py b/napalm_device_types/roles.py index dcf8815..914a7f6 100644 --- a/napalm_device_types/roles.py +++ b/napalm_device_types/roles.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Which roles a driver class fills, and which of them is primary. A *role* is what a device is: a firewall, a switch, a NAS. Real devices are @@ -22,7 +21,7 @@ when a concrete driver provided it. from __future__ import annotations -from typing import Any, List, Optional, Type +from typing import Any def _is_role_base(cls: type) -> bool: @@ -36,7 +35,7 @@ def _is_role_base(cls: type) -> bool: return isinstance(role, str) and bool(role) -def roles_of(driver_cls: type) -> List[Type[Any]]: +def roles_of(driver_cls: type) -> list[type[Any]]: """Every role base in ``driver_cls``'s MRO, most significant first. MRO order is inheritance order, so the list reflects exactly what the driver @@ -49,12 +48,12 @@ def roles_of(driver_cls: type) -> List[Type[Any]]: return [cls for cls in roles if not any(o is not cls and issubclass(o, cls) for o in roles)] -def role_keys_of(driver_cls: type) -> List[str]: +def role_keys_of(driver_cls: type) -> list[str]: """The stable string keys of :func:`roles_of`, e.g. ``["storage", "linux"]``.""" return [vars(cls)["ROLE"] for cls in roles_of(driver_cls)] -def primary_role_of(driver_cls: type) -> Optional[str]: +def primary_role_of(driver_cls: type) -> str | None: """The driver's headline role, or ``None`` when it declares no role at all. This is the value netOrk exposes as ``device_class``. diff --git a/napalm_device_types/service_clusters.py b/napalm_device_types/service_clusters.py new file mode 100644 index 0000000..c117ad7 --- /dev/null +++ b/napalm_device_types/service_clusters.py @@ -0,0 +1,538 @@ +"""Service clusters: the configuration that says which services run together. + +Two hosts that answer at one virtual IP may run the same service by accident. +What proves a cluster is the configuration its members share. That is read the +same way on every Linux host, so the command and its parse live here once, and +a driver only carries the command across (netOrk #1125). + +**What is read.** + +- *Patroni*: every ``patroni.service`` and ``patroni@*.service`` unit systemd + knows. Its configuration file -- the unit's first non-flag argument, else + ``/etc/patroni/config.yml`` -- gives ``scope``, ``name`` and + ``restapi.connect_address``. The REST API there (``/cluster``, which Patroni + answers without a login) lists the members its DCS knows, each with its role: + ``leader``, ``sync_standby``, ``replica``. When it does not answer, the host + is a member of its scope on its own. +- *adguardhome-sync*: a running container whose image is adguardhome-sync. + ``ORIGIN_URL`` and ``REPLICA*_URL`` name the AdGuard Home instances it keeps in + step: the origin pushes to every replica. A sync configured by a YAML file + instead of its environment is not read. +- *BunkerWeb*: a running instance container (``bunkerity/bunkerweb``) admits a + manager's configuration pushes from the addresses in ``API_WHITELIST_IP``; a + manager (``bunkerweb-all-in-one``, ``bunkerweb-scheduler``) names its + instances in ``BUNKERWEB_INSTANCES``. BunkerWeb's own default whitelist, + loopback and the docker network, names no one. A manager that keeps its + instances in its database names none here: its workers name it. + +**Filtered on the host.** The same configuration holds secrets -- Patroni's +superuser and REST passwords, the sync's AdGuard logins, BunkerWeb's database +URI. The command never prints a configuration: awk programs print allowlisted +fields only, and a URL loses its ``user:password@``. The DCS endpoints stay on +the host too; the members come from ``/cluster``. + +**None is "did not look".** A Patroni configuration that exists but cannot be +read, a host without systemd, a docker the login may not ask, a sync configured +by file: that mechanism is None, and the caller keeps what it knew. ``[]`` is a +host that has none. + +**Root, and without it.** Patroni's configuration is the postgres user's alone. +The whole script goes to the host as one ``sh -c`` argument; when the privileged +call brings no report back, it runs again without privilege. +""" + +from __future__ import annotations + +import ipaddress +import json +import re +from collections.abc import Callable +from shlex import quote +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +from napalm_device_types.models import ( + ServiceClusterDict, + ServiceClusterMemberDict, + ServiceClustersDict, +) +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "SVCC_BEGIN" +_END = "SVCC_END" +#: Exists only while systemd runs as PID 1; a container may carry systemctl without. +_SYSTEMD_RUN = "/run/systemd/system" +_PATRONI_CONFIG = "/etc/patroni/config.yml" +#: The container environment that may leave the host. +_ENV_KEYS = "ORIGIN_URL|REPLICA[0-9]*_URL|API_WHITELIST_IP|BUNKERWEB_INSTANCES|API_HTTP_PORT" +#: BunkerWeb's instance API, where a manager pushes its configuration. +_BUNKERWEB_API_PORT = 5000 + +#: Fed a unit's ``systemctl show`` output: ``active `` and ``config ``, +#: the first argument of the command line that is no flag. +_UNIT_AWK = " ".join( + ( + r'/^ActiveState=/ { print "active " substr($0, 13); next }', + r'/^ExecStart=/ { s = $0; i = index(s, "argv[]="); if (!i) next; s = substr(s, i + 7);' + r' j = index(s, " ;"); if (j) s = substr(s, 1, j - 1); n = split(s, t, " ");' + r' for (k = 2; k <= n; k++) if (!c && t[k] !~ /^-/) { print "config " t[k]; c = 1 } }', + ) +) + +#: Fed Patroni's YAML: ``scope``, ``name`` (top level only -- ``tags`` may carry a +#: ``name`` too), ``restapi`` (its connect_address) and ``tls yes`` when the REST +#: API has a certificate. Nothing else leaves the file. +_PATRONI_AWK = " ".join( + ( + r'function v(l) { sub(/^[^:]*:[ \t]*/, "", l); return l }', + r'/^[A-Za-z_][A-Za-z0-9_]*:/ { sec = $0; sub(/:.*/, "", sec);' + r' if (sec == "scope" || sec == "name") print sec " " v($0); next }', + r'sec == "restapi" && /^[ \t]+connect_address:/ { print "restapi " v($0) }', + r'sec == "restapi" && /^[ \t]+certfile:/ { print "tls yes" }', + ) +) + +#: Fed a container's environment, one ``KEY=value`` a line: the allowlisted keys, +#: each URL without its ``user:password@``. +_ENV_AWK = " ".join( + ( + r'{ x = index($0, "="); if (!x) next; k = substr($0, 1, x - 1); v = substr($0, x + 1);', + rf" if (k !~ /^({_ENV_KEYS})$/) next;", + r' gsub(/:\/\/[^@\/ ]*@/, "://", v); print "env " k " " v }', + ) +) + +_PATRONI_SH = ( + f"if command -v systemctl >/dev/null 2>&1 && [ -d {_SYSTEMD_RUN} ]; then echo '[systemd]'; " + "for u in $(systemctl list-units --all --plain --no-legend --type=service " + "patroni.service 'patroni@*' 2>/dev/null | awk '{print $1}'); do " + 'echo "[patroni $u]"; ' + 'o=$(systemctl show -p ActiveState -p ExecStart "$u" 2>/dev/null | ' + f"awk {quote(_UNIT_AWK)}); " + 'echo "$o"; ' + "f=$(echo \"$o\" | sed -n 's/^config //p' | head -n 1); " + f'[ -n "$f" ] || f={_PATRONI_CONFIG}; ' + 'if [ -r "$f" ]; then ' + f'p=$(awk {quote(_PATRONI_AWK)} "$f"); echo "$p"; ' + "r=$(echo \"$p\" | sed -n 's/^restapi //p' | head -n 1 | tr -d '\\042\\047 '); " + 's=http; case "$p" in *"tls yes"*) s=https;; esac; ' + 'if [ -n "$r" ]; then ' + 'b=$(curl -s -k -m 3 "$s://$r/cluster" 2>/dev/null || ' + 'wget -q -T 3 --no-check-certificate -O - "$s://$r/cluster" 2>/dev/null); ' + 'if [ -n "$b" ]; then echo "rest $(echo "$b" | tr -d \'\\n\')"; ' + "else echo rest-unreachable; fi; fi; " + 'elif [ -e "$f" ]; then echo "unreadable $f"; ' + 'else echo "missing $f"; fi; ' + "done; " + "else echo '[no-systemd]'; fi; " +) + + +def _docker_sh(binary: str | None) -> str: + """The containers part: running containers of the clustered images, their + allowlisted environment. ``[no-docker]`` for a host without the engine.""" + if not binary: + return "echo '[no-docker]'; " + return ( + f"D={quote(binary)}; " + 'if command -v "$D" >/dev/null 2>&1; then ' + "if c=$(\"$D\" ps --format '{{.Names}}' 2>/dev/null); then echo '[docker]'; " + "for n in $c; do " + 'i=$("$D" inspect --format \'{{.Config.Image}}\' "$n" 2>/dev/null); ' + 'case "$i" in *adguardhome-sync*|*bunkerweb*) ' + 'echo "[container $n]"; echo "image $i"; ' + '"$D" inspect --format \'{{range .Config.Env}}{{println .}}{{end}}\' "$n" ' + f"2>/dev/null | awk {quote(_ENV_AWK)};; esac; done; " + "else echo '[docker-unreadable]'; fi; " + "else echo '[no-docker]'; fi; " + ) + + +def service_clusters_command(docker: str | None = "docker") -> str: + """One line of POSIX ``sh``, read-only. *docker* is the engine's binary as the + driver reaches it (QNAP's lives elsewhere), None for a host without one. The + frame markers are printed in two halves so that a transport which echoes the + command does not show them early.""" + return ( + "PATH=$PATH:/usr/sbin:/sbin; " + "printf '%s%s\\n' SVCC_ BEGIN; " + + _PATRONI_SH + + _docker_sh(docker) + + "printf '%s%s\\n' SVCC_ END" + ) + + +SERVICE_CLUSTERS_COMMAND = service_clusters_command() + +_COMMENT_RE = re.compile(r"\s+#.*$") +_Record = tuple[str, str] +_Section = tuple[str, str, list[_Record]] + + +def _frame(output: str) -> list[str]: + lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] + try: + start = lines.index(_BEGIN) + end = lines.index(_END, start) + except ValueError: + raise ValueError("no intact service cluster report in the output") from None + return lines[start + 1 : end] + + +def _sections(lines: list[str]) -> list[_Section]: + """``[ ]`` headers, each with its records as ``(kind, rest)``.""" + sections: list[_Section] = [] + for line in lines: + if line.startswith("[") and line.endswith("]"): + kind, _, name = line[1:-1].partition(" ") + sections.append((kind, name, [])) + elif line and sections: + kind, _, rest = line.partition(" ") + sections[-1][2].append((kind, rest.strip())) + return sections + + +def _clean(value: str) -> str: + """Without a trailing ``# comment`` and the quotes around it.""" + value = _COMMENT_RE.sub("", value.strip()) + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + return value.strip() + + +def _member( + *, + address: str | None = None, + port: int | None = None, + name: str | None = None, + role: str | None = None, + state: str | None = None, + this_host: bool = False, + config_from: int | None = None, +) -> ServiceClusterMemberDict: + return { + "name": name, + "address": address, + "port": port, + "role": role, + "state": state, + "this_host": this_host, + "config_from": config_from, + } + + +def _cluster( + mechanism: str, + service: str, + source: str, + cluster: str | None, + members: list[ServiceClusterMemberDict], + running: bool | None = True, +) -> ServiceClusterDict: + return { + "mechanism": mechanism, + "service": service, + "source": source, + "cluster": cluster, + "running": running, + "members": members, + } + + +def _host_port(text: str) -> tuple[str | None, int | None]: + """``"10.0.0.5:8008"`` or ``"[fd00::5]:8008"`` -> host and port.""" + try: + split = urlsplit(f"//{text.strip()}") + return split.hostname, split.port + except ValueError: + return None, None + + +def _url(text: str) -> tuple[str | None, int | None]: + """A URL's host and port, the scheme's default where it names none.""" + value = text.strip() + if "://" not in value: + value = f"http://{value}" + try: + split = urlsplit(value) + port = split.port or {"http": 80, "https": 443}.get(split.scheme) + except ValueError: + return None, None + return split.hostname, port + + +# --- Patroni ------------------------------------------------------------------ + + +def _patroni_members(rest: str, own: str | None) -> list[ServiceClusterMemberDict] | None: + try: + members = json.loads(rest).get("members") + except (ValueError, AttributeError): + return None + if not isinstance(members, list): + return None + found = [] + for entry in members: + if not isinstance(entry, dict) or not entry.get("name"): + continue + port = entry.get("port") + found.append( + _member( + name=str(entry["name"]), + address=str(entry["host"]) if entry.get("host") else None, + port=port if isinstance(port, int) else None, + role=str(entry["role"]) if entry.get("role") else None, + state=str(entry["state"]) if entry.get("state") else None, + this_host=entry["name"] == own, + ) + ) + return sorted(found, key=lambda m: m["name"] or "") + + +def _patroni(unit: str, records: list[_Record]) -> list[ServiceClusterDict]: + values: dict[str, str] = {} + rest: str | None = None + for kind, text in records: + if kind in ("active", "scope", "name", "restapi") and kind not in values: + values[kind] = _clean(text) + elif kind == "rest": + rest = text + scope = values.get("scope") + if not scope: + return [] + own = values.get("name") or None + members = _patroni_members(rest, own) if rest else None + if not members: + address, _ = _host_port(values.get("restapi", "")) + members = [_member(name=own, address=address, this_host=True)] + running = values["active"] == "active" if "active" in values else None + source = unit[: -len(".service")] if unit.endswith(".service") else unit + return [_cluster("patroni", "patroni", source, scope, members, running)] + + +# --- Containers ---------------------------------------------------------------- + + +def _env(records: list[_Record]) -> dict[str, str]: + env: dict[str, str] = {} + for kind, text in records: + if kind == "env": + key, _, value = text.partition(" ") + env[key] = value.strip() + return env + + +def _image_name(records: list[_Record]) -> str: + """``bunkerity/bunkerweb@sha256:...`` -> ``bunkerweb``.""" + image = next((text for kind, text in records if kind == "image"), "") + return image.split("@")[0].rsplit("/", 1)[-1].split(":")[0] + + +def _adguardhome_sync(name: str, records: list[_Record]) -> list[ServiceClusterDict] | None: + env = _env(records) + origin = _url(env["ORIGIN_URL"]) if env.get("ORIGIN_URL") else (None, None) + if origin[0] is None: + return None # configured by its YAML file, which is not read + replicas = sorted( + (key for key in env if re.fullmatch(r"REPLICA[0-9]*_URL", key)), + key=lambda key: int(key[7:-4] or 0), + ) + members = [_member(address=origin[0], port=origin[1], role="origin")] + for key in replicas: + host, port = _url(env[key]) + if host: + members.append(_member(address=host, port=port, role="replica", config_from=0)) + cluster = f"{origin[0]}:{origin[1]}" if origin[1] else origin[0] + return [_cluster("adguardhome-sync", "adguardhome", name, cluster, members)] + + +def _peer_address(entry: str) -> str | None: + """An entry of a whitelist that names one peer: a single address that is no + loopback, link-local or unspecified one -- not a network.""" + text = entry.strip() + if "/" in text: + try: + network = ipaddress.ip_network(text, strict=False) + except ValueError: + return None + if network.num_addresses != 1: + return None + text = str(network.network_address) + try: + address = ipaddress.ip_address(text) + except ValueError: + return None + if address.is_loopback or address.is_link_local or address.is_unspecified: + return None + return str(address) + + +def _api_port(env: dict[str, str]) -> int: + port = env.get("API_HTTP_PORT", "") + return int(port) if port.isdigit() else _BUNKERWEB_API_PORT + + +def _bunkerweb_worker(name: str, env: dict[str, str]) -> list[ServiceClusterDict]: + managers = [ + address + for entry in re.split(r"[\s,]+", env.get("API_WHITELIST_IP", "")) + if entry and (address := _peer_address(entry)) + ] + if not managers: + return [] + members = [_member(port=_api_port(env), role="worker", this_host=True, config_from=1)] + members += [_member(address=address, role="manager") for address in managers] + return [_cluster("bunkerweb", "bunkerweb", name, managers[0], members)] + + +def _bunkerweb_manager(name: str, env: dict[str, str]) -> list[ServiceClusterDict]: + instances = [entry for entry in env.get("BUNKERWEB_INSTANCES", "").split() if entry] + if not instances: + return [] + members = [_member(role="manager", this_host=True)] + for entry in instances: + host, port = _host_port(entry) + if host: + members.append( + _member( + address=host, port=port or _BUNKERWEB_API_PORT, role="worker", config_from=0 + ) + ) + return [_cluster("bunkerweb", "bunkerweb", name, None, members)] if len(members) > 1 else [] + + +def _bunkerweb(name: str, records: list[_Record]) -> list[ServiceClusterDict]: + image = _image_name(records) + if image == "bunkerweb": + return _bunkerweb_worker(name, _env(records)) + if image in ("bunkerweb-all-in-one", "bunkerweb-scheduler"): + return _bunkerweb_manager(name, _env(records)) + return [] # the UI, the API, autoconf: no instance and no manager + + +_CONTAINER_READERS: dict[str, Callable[[str, list[_Record]], list[ServiceClusterDict] | None]] = { + "adguardhome_sync": _adguardhome_sync, + "bunkerweb": _bunkerweb, +} + + +def _reader_for(records: list[_Record]) -> str | None: + image = next((text for kind, text in records if kind == "image"), "") + if "adguardhome-sync" in image: + return "adguardhome_sync" + if "bunkerweb" in image: + return "bunkerweb" + return None + + +def _containers(sections: list[_Section]) -> dict[str, list[ServiceClusterDict] | None]: + kinds = {kind for kind, _, _ in sections} + if "no-docker" in kinds: + return {"adguardhome_sync": [], "bunkerweb": []} + if "docker" not in kinds: + return {"adguardhome_sync": None, "bunkerweb": None} + found: dict[str, list[ServiceClusterDict] | None] = {"adguardhome_sync": [], "bunkerweb": []} + for kind, name, records in sections: + key = _reader_for(records) if kind == "container" else None + if key is None or found[key] is None: + continue + clusters = _CONTAINER_READERS[key](name, records) + if clusters is None: + found[key] = None + else: + found[key].extend(clusters) # type: ignore[union-attr] + return found + + +def parse_service_clusters(output: str) -> ServiceClustersDict: + """Parse what :func:`service_clusters_command` printed. + + :raises ValueError: when the output carries no intact report. + """ + sections = _sections(_frame(output)) + patroni: list[ServiceClusterDict] | None + if any(kind == "no-systemd" for kind, _, _ in sections): + patroni = None + elif any( + kind == "patroni" and any(r[0] == "unreadable" for r in records) + for kind, _, records in sections + ): + patroni = None + else: + patroni = [ + c + for kind, unit, records in sections + if kind == "patroni" + for c in _patroni(unit, records) + ] + containers = _containers(sections) + return { + "patroni": patroni, + "adguardhome_sync": containers["adguardhome_sync"], + "bunkerweb": containers["bunkerweb"], + } + + +class ServiceClustersMixin: + """Adds :meth:`get_service_clusters` to a driver that can run a command on a Linux host. + + The template form (README, "Function classes"): the command and its parse are + the same everywhere, so they are concrete here, and a driver supplies only + :meth:`_run_service_clusters_command` -- how a command reaches its host, and + how it gains root there. A driver with a container engine of its own path + (``ContainerEngineMixin._container_engine_binary``) has it asked with that. + Mixed in by the drivers that can, so ``hasattr(driver, + "get_service_clusters")`` stays a truthful answer. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_service_clusters_command(self, command: str, *, privileged: bool) -> str: + """Run *command* on the host and return what it printed; as root when + *privileged*. The command is a single ``sh -c`` invocation of a few kB: + send it on an exec channel, not typed into a shell.""" + ... + + def _service_clusters_docker(self) -> str: + binary = getattr(self, "_container_engine_binary", None) + return binary("docker") if callable(binary) else "docker" + + def get_service_clusters(self) -> ServiceClustersDict: + """ + Returns the clusters the host's configuration says it belongs to, per + mechanism: Patroni (its scope and the members its DCS lists), + adguardhome-sync (its origin and replicas), BunkerWeb (a worker's manager, + a manager's instances). See :class:`~napalm_device_types.models.ServiceClusterDict`. + + None for a mechanism means it could not be read; ``[]`` that the host has none. + + Example:: + + { + "patroni": [ + {"mechanism": "patroni", "service": "patroni", "source": "patroni", + "cluster": "netork-db", + "running": True, + "members": [ + {"name": "db-01", "address": "10.7.234.12", "port": 5432, + "role": "leader", "state": "running", "this_host": True, + "config_from": None}, + {"name": "db-02", "address": "10.7.234.11", "port": 5432, + "role": "sync_standby", "state": "streaming", "this_host": False, + "config_from": None}, + ]}, + ], + "adguardhome_sync": [], + "bunkerweb": [], + } + + :raises ValueError: if neither reading carried an intact report. + """ + command = f"sh -c {quote(service_clusters_command(self._service_clusters_docker()))}" + try: + return parse_service_clusters( + self._run_service_clusters_command(command, privileged=True) + ) + except ValueError: + pass + return parse_service_clusters(self._run_service_clusters_command(command, privileged=False)) diff --git a/napalm_device_types/service_packages.py b/napalm_device_types/service_packages.py new file mode 100644 index 0000000..cf438f1 --- /dev/null +++ b/napalm_device_types/service_packages.py @@ -0,0 +1,182 @@ +"""Which installed packages a systemd service is made of. + +A vulnerability is found on a *package*; a statement about a *service* -- the +postgresql unit, nginx -- reaches its findings only once it is known which +packages that service is. Here that is the package shipping the unit file and +the one shipping the binary its ``ExecStart`` runs: ``nginx-common`` and +``nginx-core``, ``postgresql-common`` and ``postgresql-16``. Reading it is the +same on every systemd host, so the command and its parse live here once and a +driver only carries the command across. + +**Only the unit's own packages.** The libraries its binary links against +(``libssl3`` under nginx) are every service's at once; a statement about nginx +is not one about libc, and following the dependencies would make it one. + +The command is read-only and needs no privileges. It prints ``systemctl show`` +for every loaded service unit and then asks the package manager about every +path at once: ``dpkg -S`` once for all of them -- one call, since each loads the +whole file list, and per path that is minutes on an ordinary host -- or +``rpm -qf`` per path. dpkg knows a file by the path its package shipped, so each +path is asked across the ``/usr`` merge too. The report is framed; one whose end +is missing raises, because a list cut short must never read as services that own +nothing. +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +_BEGIN = "SVCPKG_BEGIN" +_END = "SVCPKG_END" +_SUFFIX = ".service" + +#: A Debian or RPM package name. +_PACKAGE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9+._-]*$") +#: The binary of an ``ExecStart`` as ``systemctl show`` prints it. +_EXEC_PATH = re.compile(r"\bpath=(/[^ ;]+)") + +#: Every unit file and ``ExecStart`` binary in ``$s``, each also across the +#: ``/usr`` merge, one per line. +_PATHS = ( + "printf '%s\\n' \"$s\" | sed -n -e 's/^FragmentPath=\\(\\/.*\\)/\\1/p' " + "-e 's/^ExecStart=.*path=\\(\\/[^ ;]*\\).*/\\1/p' | sort -u " + "| sed -e 'p' -e 's#^/usr/#/#;t' -e 's#^/#/usr/#'" +) + +#: One line, POSIX ``sh``, read-only. The frame markers are printed in two +#: halves so that a transport which echoes the command does not show them early. +SERVICE_PACKAGES_COMMAND = ( + "s=$(systemctl list-units --type=service --all --no-legend --no-pager --plain 2>/dev/null " + "| awk '{print $1}' | tr '\\n' '\\0' | xargs -0 -r systemctl show --no-pager " + "-p Id -p FragmentPath -p ExecStart -- 2>/dev/null); " + "printf '%s%s\\n' SVCPKG_ BEGIN; echo '[units]'; printf '%s\\n\\n' \"$s\"; " + "if command -v dpkg >/dev/null 2>&1; then echo '[dpkg]'; " + f"{_PATHS} | tr '\\n' '\\0' | xargs -0 -r dpkg -S 2>/dev/null; " + "elif command -v rpm >/dev/null 2>&1; then echo '[rpm]'; " + f"{_PATHS} | while IFS= read -r f; do " + 'printf \'%s\\t%s\\n\' "$f" "$(rpm -qf --qf \'%{NAME} \' "$f" 2>/dev/null)"; done; fi; ' + "printf '%s%s\\n' SVCPKG_ END" +) + + +def _report(output: str) -> dict[str, list[str]]: + lines = [line.rstrip() for line in output.splitlines()] + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact service package report in the output") from None + sections: dict[str, list[str]] = {} + current: list[str] | None = None + for line in lines[start + 1 : end]: + if line in ("[units]", "[dpkg]", "[rpm]"): + current = sections.setdefault(line[1:-1], []) + elif current is not None: + current.append(line) + return sections + + +def _units(lines: list[str]) -> dict[str, set[str]]: + """``systemctl show`` blocks -> ``{unit: its unit file and binary}``.""" + units: dict[str, set[str]] = {} + unit = "" + paths: set[str] = set() + for line in [*lines, ""]: + key, _, value = line.partition("=") + if not line.strip(): + if unit: + units[unit] = paths + unit, paths = "", set() + elif key == "Id" and value.endswith(_SUFFIX): + unit = value[: -len(_SUFFIX)] + elif key == "FragmentPath" and value.startswith("/"): + paths.add(value) + elif key == "ExecStart": + match = _EXEC_PATH.search(value) + if match: + paths.add(match.group(1)) + return units + + +def _names(raw: str) -> set[str]: + """``libfoo1:amd64, foo-bin`` -> ``{"libfoo1", "foo-bin"}``; nothing else counts.""" + found = set() + for name in raw.replace(",", " ").split(): + name = name.split(":", 1)[0] + if _PACKAGE.match(name): + found.add(name) + return found + + +def _owners(sections: dict[str, list[str]]) -> dict[str, set[str]]: + """``{path: the packages owning it}`` from dpkg's or rpm's answer.""" + owners: dict[str, set[str]] = {} + for line in sections.get("dpkg", []): + # "pkg[, pkg2][:arch]: /path"; "diversion by x from: /path" is no owner. + if line.startswith("diversion by") or ": /" not in line: + continue + names, _, path = line.partition(": /") + owners.setdefault("/" + path.strip(), set()).update(_names(names)) + for line in sections.get("rpm", []): + path, _, names = line.partition("\t") + if " is not owned" not in names: + owners.setdefault(path.strip(), set()).update(_names(names)) + return owners + + +def _across_usr_merge(path: str) -> tuple[str, str]: + return (path, path[4:] if path.startswith("/usr/") else "/usr" + path) + + +def parse_service_packages(output: str) -> dict[str, list[str]]: + """Parse what :data:`SERVICE_PACKAGES_COMMAND` printed. + + ``{unit name without ".service": its packages, sorted}``. A unit no package + owns -- a unit file an admin wrote -- maps to an empty list: read, and none. + + :raises ValueError: when the output carries no intact report. + """ + sections = _report(output) + owners = _owners(sections) + return { + unit: sorted( + {name for p in paths for alt in _across_usr_merge(p) for name in owners.get(alt, ())} + ) + for unit, paths in _units(sections.get("units", [])).items() + } + + +class ServicePackagesMixin: + """Adds :meth:`get_service_packages` to a driver that can run a command on a systemd host. + + The template form, like :class:`~napalm_device_types.kernel.KernelFactsMixin`: + the reading and its parse are concrete here, and a driver supplies only + :meth:`_run_service_packages_command` -- how a command reaches its host. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_service_packages_command(self, command: str) -> str: + """Run *command* on the host with ``sh`` and return what it printed.""" + ... + + def get_service_packages(self) -> dict[str, list[str]]: + """ + Returns, per systemd service, the installed packages it is made of. + + The package that ships its unit file and the one that ships the binary + its ``ExecStart`` runs -- not the libraries that binary links against. + + Example:: + + { + "nginx": ["nginx-common", "nginx-core"], + "postgresql@16-main": ["postgresql-16", "postgresql-common"], + "local-backup": [], + } + + :raises ValueError: if the host's output carried no intact report. + """ + return parse_service_packages(self._run_service_packages_command(SERVICE_PACKAGES_COMMAND)) diff --git a/napalm_device_types/services.py b/napalm_device_types/services.py index 1c8a490..5e43ec2 100644 --- a/napalm_device_types/services.py +++ b/napalm_device_types/services.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Init-system services: listing them and starting/stopping them. Deliberately *not* the same thing as a NAS's exported shares, which live on @@ -13,7 +12,7 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ServiceDict @@ -21,7 +20,7 @@ from napalm_device_types.models import ServiceDict class ServiceControlMixin: if TYPE_CHECKING: - def get_services(self) -> List[ServiceDict]: + def get_services(self) -> list[ServiceDict]: """ Returns the list of system services known to the device's init system. @@ -42,7 +41,7 @@ class ServiceControlMixin: """ ... - def manage_service(self, name: str, action: str) -> Dict[str, Any]: + def manage_service(self, name: str, action: str) -> dict[str, Any]: """ Execute a lifecycle action on a named service. diff --git a/napalm_device_types/snmp_health_profile.py b/napalm_device_types/snmp_health_profile.py new file mode 100644 index 0000000..7a01d5f --- /dev/null +++ b/napalm_device_types/snmp_health_profile.py @@ -0,0 +1,60 @@ +"""The shape a driver states its device's SNMP health OIDs in (``SNMP_HEALTH_PROFILE``). + +Apart from ``models.py``, which is over the file size limit; ``models`` still exports them. +""" + +from typing import Literal + +from typing_extensions import TypedDict + + +class SnmpValueSpec(TypedDict, total=False): + """Where an SNMP agent publishes one value, inside a ``SnmpHealthProfile``. + + Either ``oids``, scalars asked in order (the first non-empty answer wins), or + ``table``, a column walked and folded to one value by ``fold``. ``pattern`` is a + regular expression whose first group is the number, for an agent that answers + with text. + + ``unknown`` says the agent answers this value wrongly, or not at all, where it + should: the application reports no value -- neither the standard reading nor a + default. It stands alone, without ``oids`` or ``table``. + """ + + oids: list[str] + table: str + fold: Literal["first", "average", "max"] + pattern: str + unknown: bool + + +class SnmpMemorySpec(TypedDict, total=False): + """Memory: ``total`` with ``used`` or ``free``, or ``percent`` alone.""" + + total: SnmpValueSpec + used: SnmpValueSpec + free: SnmpValueSpec + percent: SnmpValueSpec + unit: Literal["KiB", "bytes"] # of total, used and free; KiB, as UCD-MIB counts, by default + + +class SnmpInterfacesSpec(TypedDict, total=False): + """How the IF-MIB counters of a device are to be read.""" + + skip: str # a pattern of interface names whose counters are noise, not signal + tx_errors_are_drops: bool # the device counts dropped packets as TX errors (OPNsense) + + +class SnmpHealthProfile(TypedDict, total=False): + """Where a device's SNMP agent publishes its health values. + + A driver declares it as ``SNMP_HEALTH_PROFILE`` when its device does not answer + the standard UCD-MIB and IF-MIB; the application polls and computes. What a + profile does not name is read the standard way, and what it marks ``unknown`` is + not read at all. Uptime is in TimeTicks, CPU in percent busy. + """ + + uptime: SnmpValueSpec + cpu: SnmpValueSpec + memory: SnmpMemorySpec + interfaces: SnmpInterfacesSpec diff --git a/napalm_device_types/ssh_host_keys.py b/napalm_device_types/ssh_host_keys.py new file mode 100644 index 0000000..4413cf6 --- /dev/null +++ b/napalm_device_types/ssh_host_keys.py @@ -0,0 +1,354 @@ +"""SSH host keys: trust on first use, and a changed key refused before any login. + +A driver that accepts whatever host key answers at a device's address logs in +there with the device's credentials, whoever that is (NetOrk/netork#1167). The +layer above keeps the store of the keys it trusts; a driver only verifies: + +* It receives the keys to expect in ``optional_args["ssh_host_keys"]`` + (:data:`KNOWN_KEYS_ARG`), by the port they were seen on: + ``{"22": {"key_type": "ssh-ed25519", "key": ""}}``, the two fields of + a ``known_hosts`` line. A driver takes the one for the port it connects to, so + the caller need not know which port that is (Proxmox: its API on 8006, SSH on + 22). With a key, any other key is refused before a password or a private key + leaves the caller. Without one, the first key is accepted: trust on first use. +* It reports every key it sees through ``optional_args["ssh_host_key_observer"]`` + (:data:`OBSERVER_ARG`), a callable that receives one dict per connection:: + + {"host": str, "port": int, + "seen": {"key_type", "key", "fingerprint"}, + "expected": {...} | None, + "accepted": bool} + + ``accepted`` is False only for a refused key; the connection then raises + :class:`SshHostKeyMismatch`. A failing observer is logged and ignored, so the + store being away never stops a connection that verified. + +A caller that passes neither keeps working as before (first use, every time), +and a driver that ignores both still connects for a caller that passes them: +old and new versions on either side do not break each other. + +A connection sshd turns away before its banner -- OpenSSH 9.8+ does that to a +source it has just penalised -- is tried again after a pause +(:func:`retry_refused`, :data:`REFUSED_RETRY_WAITS`); :meth:`HostKeyCheck.connect` +does so itself, a netmiko driver wraps its ``ConnectHandler`` in it. + +:class:`HostKeyCheck` is the one implementation, for a paramiko ``SSHClient`` +(:meth:`~HostKeyCheck.connect`) and for netmiko (:meth:`~HostKeyCheck.netmiko`, +then :meth:`~HostKeyCheck.observe`). Like :mod:`napalm_device_types.channel` it +works on the objects paramiko hands it and imports no SSH library itself. +""" + +from __future__ import annotations + +import base64 +import hashlib +import logging +import os +import tempfile +import time +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any + +from napalm.base.exceptions import ConnectionException + +logger = logging.getLogger(__name__) + +#: The ``optional_args`` key that carries the host keys to expect, by port. +KNOWN_KEYS_ARG = "ssh_host_keys" +#: The ``optional_args`` key that carries the callable every seen key goes to. +OBSERVER_ARG = "ssh_host_key_observer" + +#: The waits before each new attempt when sshd turns a connection away before its +#: banner. OpenSSH 9.8+ does that to every connection from a source it has just +#: penalised (``PerSourcePenalties``) -- one whose connections did not +#: authenticate, a scanner's probes -- for seconds to minutes. 26 s in all +#: outlasts the 16 s penalty seen on a new Ubuntu 26.04 host (NetOrk/netork#1800). +REFUSED_RETRY_WAITS = (1.0, 3.0, 7.0, 15.0) + +#: The netmiko arguments this module decides; a caller's own values give way. +_NETMIKO_HOST_KEY_ARGS = ("ssh_strict", "system_host_keys", "alt_host_keys", "alt_key_file") + +_DEFAULT_SSH_PORT = 22 + + +@dataclass(frozen=True) +class SshHostKey: + """A host's public key, as the two fields of a ``known_hosts`` line.""" + + key_type: str + #: The public key blob, base64. + key: str + + @property + def fingerprint(self) -> str: + """``SHA256:…``, as ``ssh-keygen -l`` and the ssh client print it.""" + digest = hashlib.sha256(base64.b64decode(self.key)).digest() + return "SHA256:" + base64.b64encode(digest).decode("ascii").rstrip("=") + + def as_dict(self) -> dict[str, str]: + return {"key_type": self.key_type, "key": self.key, "fingerprint": self.fingerprint} + + @classmethod + def from_value(cls, value: Any) -> SshHostKey | None: + """The key in *value* (a mapping with ``key_type`` and ``key``), or None for none.""" + if not value: + return None + if not isinstance(value, Mapping) or not value.get("key_type") or not value.get("key"): + raise ValueError(f"not an SSH host key: {value!r}") + return cls(str(value["key_type"]), str(value["key"])) + + @classmethod + def of(cls, pkey: Any) -> SshHostKey: + """The key of a paramiko ``PKey`` (what a transport says the server presented).""" + return cls(pkey.get_name(), pkey.get_base64()) + + def known_hosts_line(self, host: str, port: int) -> str: + """The line under which an ssh client looks this host up.""" + name = host if port == _DEFAULT_SSH_PORT else f"[{host}]:{port}" + return f"{name} {self.key_type} {self.key}" + + +@dataclass(frozen=True) +class HostKeyObservation: + """The key one connection saw, against the key it was to expect.""" + + host: str + port: int + seen: SshHostKey + expected: SshHostKey | None + + @property + def accepted(self) -> bool: + """True on first use and for the expected key; False for a refused one.""" + return self.expected is None or self.expected == self.seen + + def as_dict(self) -> dict[str, Any]: + return { + "host": self.host, + "port": self.port, + "seen": self.seen.as_dict(), + "expected": self.expected.as_dict() if self.expected else None, + "accepted": self.accepted, + } + + +class SshHostKeyMismatch(ConnectionException): # type: ignore[misc] + """The device presented another host key than the one it is known by. + + Raised before any authentication: no password and no private key went to it. + """ + + def __init__(self, observation: HostKeyObservation) -> None: + self.observation = observation + expected = observation.expected + trusted = f"{expected.fingerprint} ({expected.key_type})" if expected else "none" + super().__init__( + f"SSH host key mismatch at {observation.host}:{observation.port}: it presented " + f"{observation.seen.fingerprint} ({observation.seen.key_type}), the trusted key is " + f"{trusted}. Nothing was sent to it." + ) + + +class _AcceptFirst: + """A paramiko missing-host-key policy for first use: accept, keep nothing. + + The key is reported once the connection stands (:meth:`HostKeyCheck.observe`); + nothing is written to a ``known_hosts`` file. + """ + + def missing_host_key(self, client: Any, hostname: str, key: Any) -> None: + return None + + +class _RefuseUnknown: + """A paramiko policy for a host with a known key: refuse anything else. + + paramiko compares a loaded key itself and raises ``BadHostKeyException`` + for another one; this only answers if the loaded key was not found under + the name connected to, which must not turn into an accepted key. + """ + + def missing_host_key(self, client: Any, hostname: str, key: Any) -> None: + raise ConnectionException(f"SSH host key of {hostname} is not the one expected") + + +def _expected_key(known: Any, port: int) -> SshHostKey | None: + """The key in *known* (``{port: key}``, the port as text or number) for *port*.""" + if not known: + return None + if not isinstance(known, Mapping): + raise ValueError(f"SSH host keys must be given by port, not as {type(known).__name__}") + return SshHostKey.from_value(known.get(str(port)) or known.get(port)) + + +def _refused_key(exc: BaseException) -> Any | None: + """paramiko's ``BadHostKeyException`` in *exc*'s chain (netmiko and drivers wrap it).""" + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + if hasattr(current, "expected_key") and hasattr(current, "key"): + return current + stack.extend([current.__cause__, current.__context__]) + return None + + +def refused_before_banner(exc: BaseException) -> bool: + """Whether *exc*, or what it wraps, is sshd closing the connection before its banner. + + paramiko says "Error reading SSH protocol banner" (netmiko and drivers wrap + it), with the connection's EOF or reset underneath. A failed login, a + silent host and a closed port are something else: waiting does not end them. + """ + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + if "protocol banner" in str(current): + return True + stack.extend([current.__cause__, current.__context__]) + return False + + +def retry_refused[T]( + attempt: Callable[[], T], + *, + waits: tuple[float, ...] = REFUSED_RETRY_WAITS, + sleep: Callable[[float], None] | None = None, +) -> T: + """*attempt*, again after each of *waits* while sshd turns it away before its banner. + + Any other error, and the last refusal, go to the caller unchanged. + """ + pause = sleep or time.sleep + for wait in waits: + try: + return attempt() + except Exception as exc: + if not refused_before_banner(exc): + raise + logger.info("SSH connection turned away before its banner; again in %s s", wait) + pause(wait) + return attempt() + + +class HostKeyCheck: + """How one driver connection verifies its host's key. + + *optional_args* are the driver's (both keys optional), *host* and *port* the + address the connection goes to, as given to paramiko or netmiko. + """ + + def __init__(self, optional_args: Mapping[str, Any] | None, host: str, port: int) -> None: + args = optional_args or {} + self.host = host + self.port = int(port) + self.expected = _expected_key(args.get(KNOWN_KEYS_ARG), self.port) + self._observer = args.get(OBSERVER_ARG) + + # ------------------------------------------------------------------ + # paramiko + # ------------------------------------------------------------------ + + def prepare(self, client: Any) -> None: + """Before ``client.connect()``: the known key, or acceptance of the first one. + + The known key goes into the client's host keys, so paramiko asks the + server for that key type first and refuses any other key itself. + """ + if self.expected is None: + client.set_missing_host_key_policy(_AcceptFirst()) + return + with self._known_hosts_file(self.expected) as path: + client.get_host_keys().load(path) + client.set_missing_host_key_policy(_RefuseUnknown()) + + def connect(self, client: Any, **kwargs: Any) -> HostKeyObservation: + """``client.connect()`` to this host with *kwargs*, verified and reported.""" + self.prepare(client) + with self.refusing(): + retry_refused(lambda: client.connect(hostname=self.host, port=self.port, **kwargs)) + return self.observe(client.get_transport()) + + # ------------------------------------------------------------------ + # netmiko + # ------------------------------------------------------------------ + + @contextmanager + def netmiko(self, base: Mapping[str, Any]) -> Iterator[dict[str, Any]]: + """``ConnectHandler`` keyword arguments: *base* with the host key decided. + + Open the connection inside the block and call :meth:`observe` on its + transport after it. A known key goes to netmiko in a ``known_hosts`` + file that exists only for the block, with ``ssh_strict``: should the + file not be read, netmiko refuses rather than accepts. + """ + kwargs = {k: v for k, v in base.items() if k not in _NETMIKO_HOST_KEY_ARGS} + kwargs["system_host_keys"] = False + if self.expected is None: + kwargs.update(ssh_strict=False, alt_host_keys=False) + with self.refusing(): + yield kwargs + return + with self._known_hosts_file(self.expected) as path: + kwargs.update(ssh_strict=True, alt_host_keys=True, alt_key_file=path) + with self.refusing(): + yield kwargs + + # ------------------------------------------------------------------ + # both + # ------------------------------------------------------------------ + + @contextmanager + def refusing(self) -> Iterator[None]: + """Around a connect: a refused host key becomes :class:`SshHostKeyMismatch`.""" + try: + yield + except Exception as exc: + refused = _refused_key(exc) + if refused is None: + raise + expected = self.expected or SshHostKey.of(refused.expected_key) + offered = SshHostKey.of(refused.key) + observation = HostKeyObservation(self.host, self.port, offered, expected) + self._report(observation) + raise SshHostKeyMismatch(observation) from exc + + def observe(self, transport: Any) -> HostKeyObservation: + """After a connect: the key the server presented on *transport*, reported.""" + seen = SshHostKey.of(transport.get_remote_server_key()) + observation = HostKeyObservation(self.host, self.port, seen, self.expected) + self._report(observation) + return observation + + def _report(self, observation: HostKeyObservation) -> None: + if not callable(self._observer): + return + try: + self._observer(observation.as_dict()) + except Exception: # the store is the caller's; a connection does not hang on it + logger.warning( + "Could not report the SSH host key of %s:%s", self.host, self.port, exc_info=True + ) + + @contextmanager + def _known_hosts_file(self, expected: SshHostKey) -> Iterator[str]: + """A private ``known_hosts`` file holding only *expected*.""" + fd, path = tempfile.mkstemp(prefix="known_hosts-") + try: + with os.fdopen(fd, "w") as f: + f.write(expected.known_hosts_line(self.host, self.port) + "\n") + yield path + finally: + try: + os.unlink(path) + except OSError: + pass diff --git a/napalm_device_types/ssh_server.py b/napalm_device_types/ssh_server.py index 16857d7..6c750bf 100644 --- a/napalm_device_types/ssh_server.py +++ b/napalm_device_types/ssh_server.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """OpenSSH server hardening: taking algorithms out of what sshd offers. A scanner such as ssh-audit names algorithms a server should stop offering -- @@ -31,20 +30,32 @@ Only without such an ``Include`` is sshd_config itself rewritten, after a copy. 3. After the reload, *verify* -- the caller's -- opens a new session. The running one survives a reload, so if the new one cannot get in, this one puts the old state back and reloads again. + +**The managing client is exempt from per-source penalties** where sshd has them +(OpenSSH 9.8+, :meth:`OpenSshServerMixin.exempt_from_ssh_penalties`). sshd +penalises a source whose connections do not authenticate -- a scanner's probes +-- and then drops every connection from it, the managing session's included. +The address is the one the host sees (``$SSH_CONNECTION``); it joins the +host's own ``PerSourcePenaltyExemptList`` in :data:`PENALTY_DROPIN_PATH`, with +the same three checks. An older sshd is not touched: it refuses the directive. """ from __future__ import annotations +import ipaddress import time +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field from shlex import quote -from typing import Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, TYPE_CHECKING +from typing import ( + TYPE_CHECKING, +) if TYPE_CHECKING: # pragma: no cover - for type checkers only from napalm_device_types.channel import CommandResult #: The algorithm classes: key -> (``sshd -T`` keyword, sshd_config directive). -SSH_ALGORITHM_CLASSES: Dict[str, Tuple[str, str]] = { +SSH_ALGORITHM_CLASSES: dict[str, tuple[str, str]] = { "kex": ("kexalgorithms", "KexAlgorithms"), "hostkey": ("hostkeyalgorithms", "HostKeyAlgorithms"), "cipher": ("ciphers", "Ciphers"), @@ -55,6 +66,9 @@ MAIN_CONFIG = "/etc/ssh/sshd_config" DROPIN_DIR = "/etc/ssh/sshd_config.d" #: ``00-`` so it sorts before every other drop-in: sshd keeps the first value it reads. DROPIN_PATH = f"{DROPIN_DIR}/00-netork-ssh-hardening.conf" +#: Where :meth:`OpenSshServerMixin.exempt_from_ssh_penalties` names the managing +#: client; ``00-`` like the hardening drop-in, since sshd keeps the first value. +PENALTY_DROPIN_PATH = f"{DROPIN_DIR}/00-netork-penalty-exempt.conf" #: What marks a line this module disabled in sshd_config. DISABLED_PREFIX = "# netork: " @@ -63,9 +77,14 @@ _HEADER = ( "# taken out of what this server offers. Delete this file to undo it.\n" ) +_PENALTY_HEADER = ( + "# Written by netOrk: the address netOrk manages this host from, exempt from\n" + "# sshd's per-source penalties (PerSourcePenalties). Delete this file to undo it.\n" +) + #: Host key type, as ``ssh-keygen -l`` prints it -> the algorithms that key serves. #: Certificate algorithms are left out: they also need a HostCertificate. -_KEY_TYPE_ALGORITHMS: Dict[str, Tuple[str, ...]] = { +_KEY_TYPE_ALGORITHMS: dict[str, tuple[str, ...]] = { "ED25519": ("ssh-ed25519",), "RSA": ("rsa-sha2-512", "rsa-sha2-256", "ssh-rsa"), "DSA": ("ssh-dss",), @@ -82,7 +101,7 @@ _RELOAD = ( ) -def _directive_key(line: str) -> Optional[str]: +def _directive_key(line: str) -> str | None: """The algorithm class a global sshd_config line sets, if it sets one.""" parts = line.split(None, 1) if not parts: @@ -94,12 +113,12 @@ def _directive_key(line: str) -> Optional[str]: return None -def parse_sshd_effective(text: str) -> Dict[str, List[str]]: +def parse_sshd_effective(text: str) -> dict[str, list[str]]: """The four algorithm lists from ``sshd -T`` output, in the server's order. :raises ValueError: if a list is missing -- not sshd -T output, or cut short. """ - found: Dict[str, List[str]] = {} + found: dict[str, list[str]] = {} for raw in text.splitlines(): parts = raw.strip().split(None, 1) if len(parts) != 2: @@ -113,9 +132,9 @@ def parse_sshd_effective(text: str) -> Dict[str, List[str]]: return found -def parse_sshd_hostkeys(text: str) -> List[str]: +def parse_sshd_hostkeys(text: str) -> list[str]: """The ``hostkey`` paths from ``sshd -T`` output, in order.""" - paths: List[str] = [] + paths: list[str] = [] for raw in text.splitlines(): parts = raw.strip().split(None, 1) if len(parts) == 2 and parts[0].lower() == "hostkey": @@ -123,12 +142,12 @@ def parse_sshd_hostkeys(text: str) -> List[str]: return paths -def host_key_algorithms(fingerprints: str) -> Optional[Set[str]]: +def host_key_algorithms(fingerprints: str) -> set[str] | None: """The host key algorithms the keys in ``ssh-keygen -l`` output can serve. None when a line is not understood or there is no key: then nothing is known. """ - usable: Set[str] = set() + usable: set[str] = set() for raw in fingerprints.splitlines(): parts = raw.split() if not parts: @@ -163,13 +182,13 @@ class Refusal: @dataclass(frozen=True) class SshHardeningPlan: - keep: Dict[str, List[str]] - remove: Dict[str, List[str]] - refused: List[Refusal] = field(default_factory=list) - not_offered: List[Refusal] = field(default_factory=list) + keep: dict[str, list[str]] + remove: dict[str, list[str]] + refused: list[Refusal] = field(default_factory=list) + not_offered: list[Refusal] = field(default_factory=list) @property - def changed_classes(self) -> List[str]: + def changed_classes(self) -> list[str]: return [k for k in SSH_ALGORITHM_CLASSES if self.remove.get(k)] @@ -177,8 +196,8 @@ def plan_ssh_hardening( effective: Mapping[str, Sequence[str]], remove: Mapping[str, Iterable[str]], *, - client: Optional[Mapping[str, Iterable[str]]] = None, - host_keys: Optional[Iterable[str]] = None, + client: Mapping[str, Iterable[str]] | None = None, + host_keys: Iterable[str] | None = None, ) -> SshHardeningPlan: """What the lists become when *remove* is taken out of *effective*. @@ -199,10 +218,10 @@ def plan_ssh_hardening( unknown = [k for k in remove if k not in SSH_ALGORITHM_CLASSES] if unknown: raise ValueError(f"unknown SSH algorithm class: {', '.join(unknown)}") - keep: Dict[str, List[str]] = {} - removed: Dict[str, List[str]] = {} - refused: List[Refusal] = [] - not_offered: List[Refusal] = [] + keep: dict[str, list[str]] = {} + removed: dict[str, list[str]] = {} + refused: list[Refusal] = [] + not_offered: list[Refusal] = [] usable_keys = set(host_keys) if host_keys is not None else None for kind in SSH_ALGORITHM_CLASSES: offered = list(effective.get(kind, [])) @@ -245,7 +264,7 @@ def render_dropin(keep: Mapping[str, Sequence[str]], kinds: Iterable[str]) -> st return _HEADER + "".join(lines) -def dropin_classes(text: str) -> List[str]: +def dropin_classes(text: str) -> list[str]: """The classes an existing drop-in sets.""" return [k for k in (_directive_key(line.strip()) for line in text.splitlines()) if k] @@ -268,11 +287,11 @@ def has_dropin_include(text: str) -> bool: return False -def _split_our_block(text: str) -> Tuple[List[str], str]: +def _split_our_block(text: str) -> tuple[list[str], str]: """(the classes an earlier rewrite put above the file, the file without them).""" if not text.startswith(_HEADER): return [], text - lines = text[len(_HEADER):].splitlines(keepends=True) + lines = text[len(_HEADER) :].splitlines(keepends=True) ours = 0 while ours < len(lines) and _directive_key(lines[ours].strip()): ours += 1 @@ -288,7 +307,7 @@ def render_main_config(text: str, keep: Mapping[str, Sequence[str]], kinds: Iter """ earlier, body = _split_our_block(text) wanted = set(kinds) | set(earlier) - out: List[str] = [] + out: list[str] = [] in_match = False for raw in body.splitlines(): if _is_match(raw.strip()): @@ -300,6 +319,56 @@ def render_main_config(text: str, keep: Mapping[str, Sequence[str]], kinds: Iter return render_dropin(keep, wanted) + "\n".join(out) + ("\n" if body.endswith("\n") else "") +def parse_penalty_exemptions(text: str) -> list[str] | None: + """The ``persourcepenaltyexemptlist`` from ``sshd -T`` output; None without penalties. + + Per-source penalties came with OpenSSH 9.8. An older sshd prints neither + line and refuses ``PerSourcePenaltyExemptList`` outright -- it does not + start with it in its configuration -- so None means: write nothing. + """ + penalties = False + exempt: list[str] = [] + for raw in text.splitlines(): + parts = raw.strip().split(None, 1) + if not parts: + continue + word = parts[0].lower() + if word == "persourcepenalties": + penalties = True + elif word == "persourcepenaltyexemptlist" and len(parts) == 2: + value = parts[1].strip() + exempt = [] if value.lower() == "none" else [a for a in value.split(",") if a] + return exempt if penalties else None + + +def client_address(ssh_connection: str) -> str | None: + """The client's address in ``$SSH_CONNECTION`` (``client port server port``), or None.""" + parts = ssh_connection.split() + if len(parts) != 4: + return None + try: + return str(ipaddress.ip_address(parts[0])) + except ValueError: + return None + + +def is_exempt(address: str, exempt: Iterable[str]) -> bool: + """Whether *address* is one of *exempt*'s addresses or inside one of its networks.""" + ip = ipaddress.ip_address(address) + for entry in exempt: + try: + if ip in ipaddress.ip_network(entry, strict=False): + return True + except ValueError: + continue + return False + + +def render_penalty_dropin(exempt: Sequence[str]) -> str: + """The drop-in stating the full exempt list.""" + return _PENALTY_HEADER + f"PerSourcePenaltyExemptList {','.join(exempt)}\n" + + @dataclass class SshHardeningResult: """What :meth:`OpenSshServerMixin.harden_ssh_server` did. @@ -312,11 +381,43 @@ class SshHardeningResult: plan: SshHardeningPlan applied: bool = False rolled_back: bool = False - path: Optional[str] = None - backup: Optional[str] = None + path: str | None = None + backup: str | None = None message: str = "" +@dataclass +class SshPenaltyExemption: + """What :meth:`OpenSshServerMixin.exempt_from_ssh_penalties` did. + + *address* is the one the host sees the managing client at. *applied* is + True only when sshd uses the exemption and a new session got in; + *rolled_back* says the host was changed and then put back. + """ + + address: str | None = None + applied: bool = False + rolled_back: bool = False + path: str | None = None + message: str = "" + + +#: Why an algorithm stays, by :class:`Refusal` reason. +_WHYS = { + "last": "it is the last one", + "client": "netOrk could no longer connect", + "host_keys_unknown": "the host keys could not be read", +} + + +def _unchanged( + result: SshPenaltyExemption, message: str, say: Callable[[str], None] +) -> SshPenaltyExemption: + result.message = message + say(message) + return result + + class OpenSshServerMixin: """Adds :meth:`harden_ssh_server` to a driver that can run commands as root on an OpenSSH host. @@ -336,19 +437,19 @@ class OpenSshServerMixin: *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None, + stdin: bytes | None = None, ) -> CommandResult: ... # ── the host ──────────────────────────────────────────────────────── - def _ssh_run(self, command: str, stdin: Optional[bytes] = None) -> "CommandResult": + def _ssh_run(self, command: str, stdin: bytes | None = None) -> CommandResult: return self.run_command(command, privileged=True, timeout=30, stdin=stdin) - def _ssh_read(self, path: str) -> Optional[str]: + def _ssh_read(self, path: str) -> str | None: result = self._ssh_run(f"cat -- {quote(path)}") return result.stdout if result.exit_code == 0 else None - def _ssh_write(self, path: str, content: str, mode_of: Optional[str] = None) -> None: + def _ssh_write(self, path: str, content: str, mode_of: str | None = None) -> None: """Replace *path* with *content*, as root, without root reading stdin. A driver gains root with ``sudo -S`` and puts the password in front of @@ -367,18 +468,24 @@ class OpenSshServerMixin: stdin=content.encode(), ) tmp = staged.stdout.strip() - if staged.exit_code != 0 or not tmp.startswith("/tmp/netork-ssh."): - raise RuntimeError(f"could not stage {path}: {(staged.stderr or staged.stdout).strip()}") + # A path on the device, made by mktemp under umask 077: not a local temp file. + if staged.exit_code != 0 or not tmp.startswith("/tmp/netork-ssh."): # nosec B108 + raise RuntimeError( + f"could not stage {path}: {(staged.stderr or staged.stdout).strip()}" + ) part = path + ".netork-tmp" result = self._ssh_run( # GNU stat, then BSD's; 644 for a file that does not exist yet. - f"r={quote(mode_of or path)}; m=$(stat -c %a -- \"$r\" 2>/dev/null" + f'r={quote(mode_of or path)}; m=$(stat -c %a -- "$r" 2>/dev/null' f' || stat -f %Lp -- "$r" 2>/dev/null) || m=644;' - f' install -m "$m" -- {quote(tmp)} {quote(part)} && mv -f -- {quote(part)} {quote(path)};' + f' install -m "$m" -- {quote(tmp)} {quote(part)}' + f" && mv -f -- {quote(part)} {quote(path)};" f" s=$?; rm -f -- {quote(tmp)} {quote(part)}; exit $s" ) if result.exit_code != 0: - raise RuntimeError(f"could not write {path}: {(result.stderr or result.stdout).strip()}") + raise RuntimeError( + f"could not write {path}: {(result.stderr or result.stdout).strip()}" + ) def _ssh_remove(self, path: str) -> None: self._ssh_run(f"rm -f -- {quote(path)}") @@ -389,7 +496,7 @@ class OpenSshServerMixin: if result.exit_code != 0: raise RuntimeError(f"could not reload sshd: {(result.stderr or result.stdout).strip()}") - def get_ssh_server_algorithms(self) -> Dict[str, List[str]]: + def get_ssh_server_algorithms(self) -> dict[str, list[str]]: """The algorithm lists sshd uses now (``sshd -T``), per class. :raises RuntimeError: if sshd -T fails -- no OpenSSH, or no root. @@ -402,7 +509,7 @@ class OpenSshServerMixin: raise RuntimeError(f"sshd -T failed: {(result.stderr or result.stdout).strip()}") return result.stdout - def _ssh_host_key_algorithms(self, sshd_t: str) -> Optional[Set[str]]: + def _ssh_host_key_algorithms(self, sshd_t: str) -> set[str] | None: """The host key algorithms the host has a key for; None when that cannot be read. A ``HostKey`` whose file does not exist is skipped, as sshd skips it. @@ -418,7 +525,7 @@ class OpenSshServerMixin: return None return host_key_algorithms(result.stdout) - def _sshd_accepts(self) -> Optional[str]: + def _sshd_accepts(self) -> str | None: """None when ``sshd -t`` accepts the configuration, else what it said.""" result = self._ssh_run("PATH=$PATH:/usr/sbin:/sbin sshd -t") return None if result.exit_code == 0 else (result.stderr or result.stdout).strip() @@ -429,9 +536,9 @@ class OpenSshServerMixin: self, remove: Mapping[str, Iterable[str]], *, - client: Optional[Mapping[str, Iterable[str]]] = None, - verify: Optional[Callable[[], bool]] = None, - log: Optional[Callable[[str], None]] = None, + client: Mapping[str, Iterable[str]] | None = None, + verify: Callable[[], bool] | None = None, + log: Callable[[str], None] | None = None, ) -> SshHardeningResult: """Stop the server offering the algorithms in *remove* (class -> names). @@ -445,22 +552,7 @@ class OpenSshServerMixin: :raises RuntimeError: if the current state cannot be read. """ say = log or (lambda _line: None) - say("Reading the algorithms sshd offers (sshd -T)") - sshd_t = self._sshd_effective_text() - host_keys: Optional[Set[str]] = None - if remove.get("hostkey"): - say("Reading which host keys the server has (ssh-keygen -l)") - host_keys = self._ssh_host_key_algorithms(sshd_t) - plan = plan_ssh_hardening(parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys) - whys = { - "last": "it is the last one", - "client": "netOrk could no longer connect", - "host_keys_unknown": "the host keys could not be read", - } - for r in plan.refused: - last_key = r.reason == "last" and r.kind == "hostkey" - why = "it is the last one the host has a key for" if last_key else whys[r.reason] - say(f"Keeping {r.algorithm}: {why}") + plan = self._plan_hardening(remove, client, say) kinds = plan.changed_classes result = SshHardeningResult(plan=plan) if not kinds: @@ -474,10 +566,56 @@ class OpenSshServerMixin: if main is None: raise RuntimeError(f"cannot read {MAIN_CONFIG}") restore = self._write_change(plan, main, result, say) - return self._apply_or_restore(plan, kinds, result, restore, verify, say) + return self._apply_or_restore( + result, + restore, + verify, + say, + ignored=lambda: self._algorithms_ignored(plan, result.path), + done="The server no longer offers the selected algorithms.", + ) + + def _plan_hardening( + self, + remove: Mapping[str, Iterable[str]], + client: Mapping[str, Iterable[str]] | None, + say: Callable[[str], None], + ) -> SshHardeningPlan: + """Read what sshd offers and which host keys it has; plan, and say what stays.""" + say("Reading the algorithms sshd offers (sshd -T)") + sshd_t = self._sshd_effective_text() + host_keys: set[str] | None = None + if remove.get("hostkey"): + say("Reading which host keys the server has (ssh-keygen -l)") + host_keys = self._ssh_host_key_algorithms(sshd_t) + plan = plan_ssh_hardening( + parse_sshd_effective(sshd_t), remove, client=client, host_keys=host_keys + ) + for r in plan.refused: + last_key = r.reason == "last" and r.kind == "hostkey" + why = "it is the last one the host has a key for" if last_key else _WHYS[r.reason] + say(f"Keeping {r.algorithm}: {why}") + return plan + + def _algorithms_ignored(self, plan: SshHardeningPlan, path: str | None) -> str | None: + """Why sshd does not use the new lists; None when it does.""" + now = self.get_ssh_server_algorithms() + names = [ + SSH_ALGORITHM_CLASSES[k][1] for k in plan.changed_classes if now[k] != plan.keep[k] + ] + if not names: + return None + return ( + f"sshd does not use the new {', '.join(names)}: a setting read before" + f" {path} decides them" + ) def _write_change( - self, plan: SshHardeningPlan, main: str, result: SshHardeningResult, say: Callable[[str], None] + self, + plan: SshHardeningPlan, + main: str, + result: SshHardeningResult, + say: Callable[[str], None], ) -> Callable[[], None]: """Write the new lists; return what puts the previous state back.""" if has_dropin_include(main): @@ -506,43 +644,33 @@ class OpenSshServerMixin: return restore_main - def _apply_or_restore( + def _apply_or_restore[R: (SshHardeningResult, SshPenaltyExemption)]( self, - plan: SshHardeningPlan, - kinds: List[str], - result: SshHardeningResult, + result: R, restore: Callable[[], None], - verify: Optional[Callable[[], bool]], + verify: Callable[[], bool] | None, say: Callable[[str], None], - ) -> SshHardeningResult: - def back(message: str, reload: bool) -> SshHardeningResult: - say(message) - say("Putting the previous configuration back") - try: - restore() - if reload: - self._reload_ssh_server() - except Exception as exc: # noqa: BLE001 - report it, the caller must look - result.message = f"{message}; putting the previous configuration back failed: {exc}" - say(result.message) - return result - result.rolled_back = True - result.message = message - return result + *, + ignored: Callable[[], str | None], + done: str, + ) -> R: + """Check what was written, reload, verify; put the old state back on any failure. + + *ignored* reads ``sshd -T`` after ``sshd -t`` and says why sshd does not + use the change, or None when it does; *done* is the message of success. + """ + + def back(message: str, reload: bool) -> R: + return self._put_back(result, restore, say, message, reload=reload) try: say("Checking the configuration (sshd -t)") complaint = self._sshd_accepts() if complaint: return back(f"sshd rejected the new configuration: {complaint}", reload=False) - now = self.get_ssh_server_algorithms() - ignored = [SSH_ALGORITHM_CLASSES[k][1] for k in kinds if now[k] != plan.keep[k]] - if ignored: - return back( - f"sshd does not use the new {', '.join(ignored)}: a setting read before" - f" {result.path} decides them", - reload=False, - ) + why = ignored() + if why: + return back(why, reload=False) say("Reloading sshd") self._reload_ssh_server() if verify is not None: @@ -552,6 +680,125 @@ class OpenSshServerMixin: except Exception as exc: # noqa: BLE001 - whatever went wrong, the host goes back return back(f"The change failed: {exc}", reload=True) result.applied = True - result.message = "The server no longer offers the selected algorithms." + result.message = done say(result.message) return result + + def _put_back[R: (SshHardeningResult, SshPenaltyExemption)]( + self, + result: R, + restore: Callable[[], None], + say: Callable[[str], None], + message: str, + *, + reload: bool, + ) -> R: + """Say *message*, put the previous state back, and say whether that worked.""" + say(message) + say("Putting the previous configuration back") + try: + restore() + if reload: + self._reload_ssh_server() + except Exception as exc: # noqa: BLE001 - report it, the caller must look + result.message = f"{message}; putting the previous configuration back failed: {exc}" + say(result.message) + return result + result.rolled_back = True + result.message = message + return result + + # ── the penalty exemption ─────────────────────────────────────────── + + def exempt_from_ssh_penalties( + self, + *, + verify: Callable[[], bool] | None = None, + log: Callable[[str], None] | None = None, + ) -> SshPenaltyExemption: + """Exempt the address this session comes from from sshd's per-source penalties. + + OpenSSH 9.8+ penalises a source whose connections do not authenticate -- + a scanner's probes -- and then drops every connection from it, the + managing client's included (NetOrk/netork#1800). The address is the + one the host sees (``$SSH_CONNECTION``), so NAT and relays do not + matter. The host's own exemptions stay: the new list is the one sshd + uses now plus this address. + + Only into the drop-in directory: a sshd_config that does not include it + is left alone, and the result says so. An sshd before 9.8 is not + touched -- it would not start with the directive. Checked like + :meth:`harden_ssh_server`: ``sshd -t``, ``sshd -T`` shows the address, + reload, then *verify*; on any failure the previous state goes back. + + :raises RuntimeError: if the session names no client address, or + ``sshd -T`` or sshd_config cannot be read. + """ + say = log or (lambda _line: None) + address = self._session_client_address() + result = SshPenaltyExemption(address=address) + say(f"netOrk connects from {address}") + say("Reading sshd's per-source penalties (sshd -T)") + exempt = parse_penalty_exemptions(self._sshd_effective_text()) + reason = self._penalty_exemption_unchanged(address, exempt) + if reason is not None or exempt is None: + return _unchanged(result, reason or "", say) + + restore = self._write_penalty_dropin([*exempt, address]) + result.path = PENALTY_DROPIN_PATH + say(f"Wrote {PENALTY_DROPIN_PATH}") + return self._apply_or_restore( + result, + restore, + verify, + say, + ignored=lambda: self._exemption_ignored(address), + done=f"{address} is exempt from sshd's per-source penalties.", + ) + + def _session_client_address(self) -> str: + """The address the host sees this session come from (``$SSH_CONNECTION``).""" + session = self.run_command('printf %s "$SSH_CONNECTION"', privileged=False, timeout=30) + address = client_address(session.stdout) + if address is None: + raise RuntimeError("the session names no client address ($SSH_CONNECTION)") + return address + + def _penalty_exemption_unchanged(self, address: str, exempt: list[str] | None) -> str | None: + """Why the exemption is left as it is; None when it is to be written.""" + if exempt is None: + return "Nothing to change: this sshd has no per-source penalties (before OpenSSH 9.8)." + if is_exempt(address, exempt): + return f"Nothing to change: {address} is already exempt." + main = self._ssh_read(MAIN_CONFIG) + if main is None: + raise RuntimeError(f"cannot read {MAIN_CONFIG}") + if not has_dropin_include(main): + return ( + f"Not changed: {MAIN_CONFIG} does not include {DROPIN_DIR}," + " where the exemption would go." + ) + return None + + def _exemption_ignored(self, address: str) -> str | None: + """Why sshd does not use the new exemption; None when it does.""" + if is_exempt(address, parse_penalty_exemptions(self._sshd_effective_text()) or []): + return None + return ( + "sshd does not use the new PerSourcePenaltyExemptList: a setting read before" + f" {PENALTY_DROPIN_PATH} decides it" + ) + + def _write_penalty_dropin(self, exempt: Sequence[str]) -> Callable[[], None]: + """Write the exemption list; return what puts the previous drop-in back.""" + previous = self._ssh_read(PENALTY_DROPIN_PATH) + self._ssh_run(f"mkdir -p -- {quote(DROPIN_DIR)}") + self._ssh_write(PENALTY_DROPIN_PATH, render_penalty_dropin(exempt)) + + def restore() -> None: + if previous is None: + self._ssh_remove(PENALTY_DROPIN_PATH) + else: + self._ssh_write(PENALTY_DROPIN_PATH, previous) + + return restore diff --git a/napalm_device_types/storage.py b/napalm_device_types/storage.py index 6ea52c4..74c62c4 100644 --- a/napalm_device_types/storage.py +++ b/napalm_device_types/storage.py @@ -10,9 +10,9 @@ Usage:: ... """ -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING + from napalm_device_types.base import DeviceTypeDriver -from napalm_device_types.packages import PackageManagementMixin from napalm_device_types.models import ( DiskPoolDict, LogicalVolumeDict, @@ -23,6 +23,7 @@ from napalm_device_types.models import ( StorageServiceDict, VolumeSnapshotDict, ) +from napalm_device_types.packages import PackageManagementMixin class StorageDriver(PackageManagementMixin, DeviceTypeDriver): @@ -47,7 +48,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): if TYPE_CHECKING: - def get_disks(self) -> List[PhysicalDiskDict]: + def get_disks(self) -> list[PhysicalDiskDict]: """ Returns all physical drives detected by the storage device. @@ -62,7 +63,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * rpm (int) - rotational speed; ``0`` for SSD/NVMe * temperature (int) - current temperature in Celsius; ``-1`` if unavailable * health (string) - ``"healthy"``, ``"warning"``, ``"failed"``, or ``"unknown"`` - * pool (string) - name of the pool this disk belongs to; empty string if unassigned/spare + * pool (string) - name of the pool this disk belongs to; + empty string if unassigned/spare Example:: @@ -95,7 +97,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): """ ... - def get_disk_pools(self) -> Dict[str, DiskPoolDict]: + def get_disk_pools(self) -> dict[str, DiskPoolDict]: """ Returns the disk pools (RAID arrays, ZFS pools, LVM volume groups, etc.) configured on the device. @@ -106,12 +108,14 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * type (string) - ``"zfs"``, ``"lvm"``, ``"md"``, ``"hardware-raid"``, ``"btrfs"`` * level (string) - RAID/redundancy level: ``"mirror"``, ``"raidz1"``, ``"raidz2"``, ``"raidz3"``, ``"stripe"``, ``"raid0"`` … ``"raid60"``, ``"single"`` - * status (string) - ``"online"``, ``"degraded"``, ``"faulted"``, ``"offline"``, ``"unknown"`` + * status (string) - ``"online"``, ``"degraded"``, ``"faulted"``, + ``"offline"``, ``"unknown"`` * total (int) - usable capacity in bytes * used (int) - used space in bytes * available (int) - free space in bytes * disks (list of strings) - slot identifiers of member drives - * auto_expand (bool) - whether the pool grows automatically when disks are replaced with larger ones + * auto_expand (bool) - whether the pool grows automatically when disks + are replaced with larger ones * dedup (bool) - whether deduplication is enabled * compression (string) - pool-level compression algorithm: ``"off"``, ``"lz4"``, ``"gzip"``, ``"zstd"``, etc. @@ -140,7 +144,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Logical volumes / datasets # ------------------------------------------------------------------ - def get_volumes(self) -> Dict[str, LogicalVolumeDict]: + def get_volumes(self) -> dict[str, LogicalVolumeDict]: """ Returns all logical volumes, ZFS datasets, or LUNs on the device. @@ -154,7 +158,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): * used (int) - space currently used in bytes * available (int) - space available in bytes * mountpoint (string) - local mount path; empty string for block volumes / LUNs - * compression (string) - active compression algorithm (``"off"``, ``"lz4"``, ``"zstd"``, etc.) + * compression (string) - active compression algorithm + (``"off"``, ``"lz4"``, ``"zstd"``, etc.) * dedup (bool) - whether deduplication is active on this volume * readonly (bool) - whether the volume is mounted read-only * snapshots (int) - number of snapshots currently held @@ -196,7 +201,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Shares # ------------------------------------------------------------------ - def get_shares(self) -> Dict[str, NASShareDict]: + def get_shares(self) -> dict[str, NASShareDict]: """ Returns all network shares and iSCSI targets exported by the device. @@ -244,7 +249,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Snapshots # ------------------------------------------------------------------ - def get_volume_snapshots(self, volume: str = "") -> List[VolumeSnapshotDict]: + def get_volume_snapshots(self, volume: str = "") -> list[VolumeSnapshotDict]: """ Returns volume / dataset snapshots. @@ -253,7 +258,8 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): Each entry contains: - * name (string) - snapshot name (e.g. ``"auto-2026-05-11"`` or full ``"tank/media@auto-2026-05-11"``) + * name (string) - snapshot name (e.g. ``"auto-2026-05-11"`` + or full ``"tank/media@auto-2026-05-11"``) * volume (string) - parent volume / dataset path * created (float) - creation timestamp (Unix epoch) * size (int) - bytes of unique data referenced only by this snapshot @@ -339,7 +345,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Quotas # ------------------------------------------------------------------ - def get_quotas(self) -> List[StorageQuotaDict]: + def get_quotas(self) -> list[StorageQuotaDict]: """ Returns all filesystem quotas configured on the device. @@ -380,7 +386,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Services # ------------------------------------------------------------------ - def get_storage_services(self) -> Dict[str, StorageServiceDict]: + def get_storage_services(self) -> dict[str, StorageServiceDict]: """ Returns the file-sharing and access services available on the device. @@ -443,7 +449,7 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # Replication # ------------------------------------------------------------------ - def get_replication_jobs(self) -> List[ReplicationJobDict]: + def get_replication_jobs(self) -> list[ReplicationJobDict]: """ Returns all replication / sync jobs configured on the device. @@ -493,8 +499,3 @@ class StorageDriver(PackageManagementMixin, DeviceTypeDriver): # ------------------------------------------------------------------ # Package management (plugins / extensions) # ------------------------------------------------------------------ - - - - - diff --git a/napalm_device_types/sudo_history.py b/napalm_device_types/sudo_history.py new file mode 100644 index 0000000..865adb6 --- /dev/null +++ b/napalm_device_types/sudo_history.py @@ -0,0 +1,312 @@ +"""The sudo password an earlier driver left in shell histories, and taking it back. + +A driver that typed ``echo '' | sudo -S -p "" `` into an +interactive shell left that line, password and all, in the login user's shell +history, once per command -- and where only the part before the first ``&&`` +ran as root, it left files root should own to the login user. A driver that has +stopped doing so still leaves both behind on every host it ever reached. + +:class:`SudoHistoryMixin` reads the first as a fact and takes both back. The +commands are the same on every POSIX host, so they live here once; a driver +mixes it in, supplies ``run_command`` and says what it once left behind: +:attr:`~SudoHistoryMixin.SUDO_HISTORY_MARKER` and +:attr:`~SudoHistoryMixin.SUDO_ROOT_FILES`. + +**The password never travels.** Lines are counted with ``grep -c`` and deleted +with ``sed``, both matching the marker only, as the login user who owns the +history. Nothing of a line's content is read back, and nothing goes to stdin. + +**What is deleted is gone from the disk too, as far as a file can say.** ``sed +-i`` with a backup suffix renames the old file to the backup and writes a new +one; the backup -- the old file's blocks -- is overwritten with ``shred -u``, +or removed with ``rm`` where there is no ``shred``. Backups, snapshots and an +audit log of the host may still hold the password: only rotating it ends that. + +**A file given back to root** must exist as a regular file, not be a symbolic +link, and be owned by the login user. ``chown -h`` never follows a link that +appears in between. +""" + +from __future__ import annotations + +from collections.abc import Callable +from shlex import quote +from typing import TYPE_CHECKING, ClassVar + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepDict, FixStepOutcomeDict + +if TYPE_CHECKING: # pragma: no cover - for type checkers only + from napalm_device_types.channel import CommandResult + +#: The shell history files looked at, in the login user's home. +HISTORY_FILES = (".bash_history", ".zsh_history", ".ash_history", ".sh_history") + +#: The suffix of the moment's backup ``sed -i`` makes; it carries no application's name. +HISTORY_BACKUP_SUFFIX = ".napalm-bak" + +_COUNT = 'LC_ALL=C grep -c -F -e "$m" -- "$p"' + + +class HistorySudoLinesDict(TypedDict): + """A history file holding lines with the sudo password (``get_history_sudo_lines``).""" + + #: The file, as ``~/``. + file: str + lines: int + + +class HistoryCleanupDict(TypedDict): + """What one history file held before the cleanup and holds after it.""" + + file: str + before: int + after: int + + +class SudoHistoryCleanupDict(FixResultDict): + """The result of ``remove_history_sudo_lines``: the fix, and what it changed.""" + + #: Every history file there is, with its lines before and after. + files: list[HistoryCleanupDict] + #: How many lines were deleted, over all files. + removed: int + #: The files given back to root. + owners_restored: list[str] + + +def _bre(text: str) -> str: + """*text* as a POSIX basic regular expression matching it literally, for ``sed``.""" + return "".join("\\" + c if c in "\\.[]*^$/" else c for c in text) + + +def _history_loop(body: str) -> str: + names = " ".join(HISTORY_FILES) + return f'for f in {names}; do p="$HOME/$f"; [ -f "$p" ] || continue; {body}; done' + + +def count_script(marker: str) -> str: + """Print `` `` per history file there is; exit 2 when one cannot be read.""" + return f"m={quote(marker)}; " + _history_loop( + f'n=$({_COUNT}); [ $? -le 1 ] || exit 2; printf "%s %s\\n" "$n" "$f"' + ) + + +def cleanup_script(marker: str) -> str: + """Delete the lines holding *marker* from every history file there is. + + Prints per file: ``done ``, ``link `` + (left alone), ``unreadable `` or ``failed ``. + """ + backup = f'"$p"{quote(HISTORY_BACKUP_SUFFIX)}' + return f"m={quote(marker)}; r={quote(_bre(marker))}; " + _history_loop( + f"b=$({_COUNT}); " + '[ $? -le 1 ] || { printf "unreadable %s\\n" "$f"; continue; }; ' + 'if [ -L "$p" ]; then printf "link %s %s\\n" "$b" "$f"; continue; fi; ' + 'if [ "$b" -gt 0 ]; then ' + f'LC_ALL=C sed -i{quote(HISTORY_BACKUP_SUFFIX)} -e "/$r/d" "$p"; s=$?; ' + # A sed that renamed the file away and wrote no new one: the backup is the history. + f'[ -e "$p" ] || mv -f {backup} "$p"; ' + f"[ -e {backup} ] && {{ shred -u {backup} 2>/dev/null || rm -f {backup}; }}; " + '[ $s -eq 0 ] || { printf "failed %s %s\\n" "$b" "$f"; continue; }; ' + "fi; " + f'a=$({_COUNT}); printf "done %s %s %s\\n" "$b" "$a" "$f"' + ) + + +def owners_script(user: str, paths: tuple[str, ...]) -> str: + """As root: give each of *paths* back to root:root where *user* owns it. + + Prints ``fixed `` or ``failed `` per file it changed or tried to. + """ + return ( + f"u={quote(user)}; for p in {' '.join(quote(p) for p in paths)}; do " + '[ -f "$p" ] && [ ! -L "$p" ] || continue; ' + '[ -n "$(find "$p" -prune -user "$u" 2>/dev/null)" ] || continue; ' + 'if chown -h 0:0 "$p"; then printf "fixed %s\\n" "$p"; ' + 'else printf "failed %s\\n" "$p"; fi; done' + ) + + +def _home(name: str) -> str: + return f"~/{name}" + + +def _lines(n: int) -> str: + return "1 line" if n == 1 else f"{n} lines" + + +def _done(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + before, after, name = int(rest[0]), int(rest[1]), _home(rest[2]) + removed = before - after + text = ( + f"{name}: {_lines(removed)} removed ({before} before, {after} after)" + if before + else f"{name}: no such line" + ) + return after == 0, text, {"file": name, "before": before, "after": after} + + +def _link(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + count = int(rest[0]) + return count == 0, f"{_home(rest[1])}: a symbolic link, left alone ({_lines(count)})", None + + +def _unreadable(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + return False, f"{_home(rest[0])}: cannot be read", None + + +def _failed(rest: list[str]) -> tuple[bool, str, HistoryCleanupDict | None]: + kept = _lines(int(rest[0])) + return False, f"{_home(rest[1])}: could not be rewritten ({kept} kept)", None + + +#: Per kind of line :func:`cleanup_script` prints: how many words follow, and what it says. +_CLEANUP_LINES: dict[ + str, tuple[int, Callable[[list[str]], tuple[bool, str, HistoryCleanupDict | None]]] +] = { + "done": (3, _done), + "link": (2, _link), + "unreadable": (1, _unreadable), + "failed": (2, _failed), +} + + +class SudoHistoryMixin: + """Reads and removes the sudo password an earlier driver left in shell histories. + + Mixed in by the drivers that once typed it into a shell, not by a base + class, so ``callable(getattr(driver, "get_history_sudo_lines", None))`` + stays a truthful answer. + """ + + #: What every line carries that typed the sudo password into the shell, as a + #: fixed string -- ``"| sudo -S -p "`` for ``echo '' | sudo -S -p "" + #: ``. None for a driver that never did: nothing is looked for. + SUDO_HISTORY_MARKER: ClassVar[str | None] = None + + #: Files an earlier version wrote as the login user where root should have. + SUDO_ROOT_FILES: ClassVar[tuple[str, ...]] = () + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: ... + + def get_history_sudo_lines(self) -> list[HistorySudoLinesDict]: + """The login user's history files that hold lines with the sudo password, and how many. + + A fact: counted with ``grep -c`` as the login user; no line is read. + An empty list when no file holds one, or the driver never typed it. + + :raises RuntimeError: when a history file cannot be read. + """ + marker = self.SUDO_HISTORY_MARKER + if not marker: + return [] + result = self.run_command(count_script(marker), timeout=30) + if result.exit_code != 0: + raise RuntimeError( + f"cannot count the shell history: {(result.stderr or result.stdout).strip()}" + ) + found: list[HistorySudoLinesDict] = [] + for line in result.stdout.splitlines(): + count, _, name = line.strip().partition(" ") + if count.isdigit() and int(count) and name: + found.append({"file": _home(name), "lines": int(count)}) + return found + + def remove_history_sudo_lines(self) -> SudoHistoryCleanupDict: + """Delete those lines, as the login user, and give :attr:`SUDO_ROOT_FILES` back to root. + + Two steps, ``history`` and ``owners``; the second runs when the first + failed, too. Succeeds when no history file holds such a line any more + and every file the login user owned is root's again. + """ + files: list[HistoryCleanupDict] = [] + restored: list[str] = [] + parts = [ + run_fix_steps([("history", lambda: self._clean_histories(files))]), + run_fix_steps([("owners", lambda: self._restore_owners(restored))]), + ] + steps: list[FixStepDict] = [s for part in parts for s in part["steps"]] + return { + "success": all(part["success"] for part in parts), + "steps": steps, + "output": "\n".join(f"[{s['step']}] {s['output']}".rstrip() for s in steps), + "files": files, + "removed": sum(f["before"] - f["after"] for f in files), + "owners_restored": restored, + } + + def _clean_histories(self, files: list[HistoryCleanupDict]) -> FixStepOutcomeDict: + marker = self.SUDO_HISTORY_MARKER + if not marker: + return {"ok": True, "output": "this driver never typed the sudo password"} + result = self.run_command(cleanup_script(marker), timeout=60) + said: list[str] = [] + ok = result.exit_code == 0 + for line in result.stdout.splitlines(): + ok = self._history_line(line.split(), files, said) and ok + if result.exit_code != 0: + said.append((result.stderr or result.stdout).strip()) + return {"ok": ok, "output": "\n".join(said) or "no shell history"} + + @staticmethod + def _history_line(words: list[str], files: list[HistoryCleanupDict], said: list[str]) -> bool: + """Take one line of :func:`cleanup_script` into *files* and *said*; whether it is clean.""" + kind, rest = (words[0], words[1:]) if words else ("", []) + read = _CLEANUP_LINES.get(kind) + if read is None or len(rest) != read[0]: + return True + clean, text, entry = read[1](rest) + said.append(text) + if entry is not None: + files.append(entry) + return clean + + def _restore_owners(self, restored: list[str]) -> FixStepOutcomeDict: + paths = tuple(self.SUDO_ROOT_FILES) + if not paths: + return {"ok": True, "output": "no file to check"} + who = self.run_command("id -u; id -un", timeout=30).stdout.split() + if len(who) < 2: + return {"ok": False, "output": "cannot tell who the login user is"} + uid, user = who[0], who[1] + if uid == "0": + return {"ok": True, "output": "logged in as root: every file it wrote is root's"} + result = self.run_command(owners_script(user, paths), privileged=True, timeout=30) + said: list[str] = [] + ok = result.exit_code == 0 + for line in result.stdout.splitlines(): + kind, _, path = line.strip().partition(" ") + if kind == "fixed": + restored.append(path) + said.append(f"{path}: owned by root again") + elif kind == "failed": + ok = False + said.append(f"{path}: could not be given back to root") + if result.exit_code != 0: + said.append((result.stderr or result.stdout).strip()) + return {"ok": ok, "output": "\n".join(said) or f"no file owned by {user}"} + + +__all__ = [ + "HISTORY_BACKUP_SUFFIX", + "HISTORY_FILES", + "HistoryCleanupDict", + "HistorySudoLinesDict", + "SudoHistoryCleanupDict", + "SudoHistoryMixin", + "cleanup_script", + "count_script", + "owners_script", +] diff --git a/napalm_device_types/switch.py b/napalm_device_types/switch.py index 07eee15..704c4f2 100644 --- a/napalm_device_types/switch.py +++ b/napalm_device_types/switch.py @@ -10,7 +10,9 @@ Usage:: ... """ -from typing import Any, Awaitable, Callable, Dict, List, TYPE_CHECKING +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any + from napalm_device_types.base import DeviceTypeDriver from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.models import ( @@ -45,7 +47,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): async def get_health_metrics( cls, snmp_get: Callable[..., Awaitable[Any]], - snmp_walk: Callable[..., Awaitable[Dict[str, Any]]], + snmp_walk: Callable[..., Awaitable[dict[str, Any]]], ) -> HealthMetricsDict: """Collect SNMP health metrics for this switch type. @@ -54,7 +56,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def get_spanning_tree(self) -> Dict[str, SpanningTreeDict]: + def get_spanning_tree(self) -> dict[str, SpanningTreeDict]: """ Returns spanning tree status for each STP instance. @@ -104,7 +106,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def get_port_channels(self) -> Dict[str, PortChannelDict]: + def get_port_channels(self) -> dict[str, PortChannelDict]: """ Returns port-channel (LAG) configuration and status. @@ -129,8 +131,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - - def get_dot1x_ports(self) -> Dict[str, Dot1XPortDict]: + def get_dot1x_ports(self) -> dict[str, Dot1XPortDict]: """ Returns the 802.1X / NAC configuration per switch port. @@ -350,7 +351,7 @@ class SwitchDriver(MacAclMixin, DeviceTypeDriver): """ ... - def set_lag_members(self, lag_name: str, members: List[str]) -> None: + def set_lag_members(self, lag_name: str, members: list[str]) -> None: """ Sets the full member-port list of a LAG/trunk interface. diff --git a/napalm_device_types/system_identity.py b/napalm_device_types/system_identity.py index 045c8a2..6549d3b 100644 --- a/napalm_device_types/system_identity.py +++ b/napalm_device_types/system_identity.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The SMBIOS identity of a host: system UUID, product serial, board serial. What ties a server to its BMC. The BMC reports the server's serial number and @@ -32,19 +31,37 @@ _DMI = "/sys/class/dmi/id" #: output cannot shift a field. SYSTEM_IDENTITY_COMMAND = ( f"printf '{_BEGIN}\\n%s\\n%s\\n%s\\n{_END}\\n' " - f"\"$(cat {_DMI}/product_uuid 2>/dev/null)\" " - f"\"$(cat {_DMI}/product_serial 2>/dev/null)\" " - f"\"$(cat {_DMI}/board_serial 2>/dev/null)\"" + f'"$(cat {_DMI}/product_uuid 2>/dev/null)" ' + f'"$(cat {_DMI}/product_serial 2>/dev/null)" ' + f'"$(cat {_DMI}/board_serial 2>/dev/null)"' ) #: Lowercased defaults board vendors leave in place. -_PLACEHOLDERS = frozenset({ - "", "none", "n/a", "na", "not specified", "not applicable", "not available", - "to be filled by o.e.m.", "to be filled by oem", "default string", "default", - "system serial number", "base board serial number", "chassis serial number", - "serial number", "0123456789", "1234567890", "123456789", "o.e.m.", "oem", - "03000200-0400-0500-0006-000700080009", -}) +_PLACEHOLDERS = frozenset( + { + "", + "none", + "n/a", + "na", + "not specified", + "not applicable", + "not available", + "to be filled by o.e.m.", + "to be filled by oem", + "default string", + "default", + "system serial number", + "base board serial number", + "chassis serial number", + "serial number", + "0123456789", + "1234567890", + "123456789", + "o.e.m.", + "oem", + "03000200-0400-0500-0006-000700080009", + } +) #: A value of one repeated character ("0", "00000000", "FFFF-...") or of zeros and Fs. _REPEATED = re.compile(r"^(.)\1*$") diff --git a/napalm_device_types/systemd.py b/napalm_device_types/systemd.py index d181d4b..1089818 100644 --- a/napalm_device_types/systemd.py +++ b/napalm_device_types/systemd.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """systemd services: listing them in one round trip, and starting and stopping them. What systemd reports about its services, and how one is started or stopped, is @@ -31,7 +30,7 @@ from __future__ import annotations import re from shlex import quote -from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from napalm_device_types.models import ServiceDict from napalm_device_types.services import ServiceControlMixin @@ -131,7 +130,7 @@ def service_action_command(name: str, action: str) -> str: ) -def parse_action_result(output: str) -> Dict[str, Any]: +def parse_action_result(output: str) -> dict[str, Any]: """``{"success", "output"}`` from what :func:`service_action_command` printed. Only the exit status decides. A job still running when ``timeout`` gave up @@ -151,7 +150,7 @@ def parse_action_result(output: str) -> Dict[str, Any]: return {"success": False, "output": text or f"systemctl exited with status {status}."} -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -161,9 +160,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> Dict[str, List[str]]: - sections: Dict[str, List[str]] = {} - current: List[str] = [] +def _sections(lines: list[str]) -> dict[str, list[str]]: + sections: dict[str, list[str]] = {} + current: list[str] = [] for line in lines: if line.startswith("[") and line.endswith("]"): current = sections.setdefault(line[1:-1], []) @@ -172,15 +171,15 @@ def _sections(lines: List[str]) -> Dict[str, List[str]]: return sections -def _unit_blocks(lines: List[str]) -> List[Dict[str, str]]: +def _unit_blocks(lines: list[str]) -> list[dict[str, str]]: """``systemctl show``'s output, one dict per unit. Units are separated by a blank line -- except where ``xargs`` split the list over two runs and the blocks meet, so a key seen twice starts the next unit as well. """ - blocks: List[Dict[str, str]] = [] - current: Dict[str, str] = {} + blocks: list[dict[str, str]] = [] + current: dict[str, str] = {} for line in lines: key, sep, value = line.partition("=") if not sep or key in current: @@ -198,17 +197,17 @@ def _base(unit: str) -> str: return unit[: -len(_SUFFIX)] -def _main_pid(block: Dict[str, str]) -> int: +def _main_pid(block: dict[str, str]) -> int: try: return int(block.get("MainPID") or 0) except ValueError: return 0 -def _loaded(blocks: List[Dict[str, str]]) -> Tuple[Dict[str, ServiceDict], Set[str]]: +def _loaded(blocks: list[dict[str, str]]) -> tuple[dict[str, ServiceDict], set[str]]: """The loaded services, and every name they go by (aliases included).""" - services: Dict[str, ServiceDict] = {} - names: Set[str] = set() + services: dict[str, ServiceDict] = {} + names: set[str] = set() for block in blocks: unit = block.get("Id", "") if not unit.endswith(_SUFFIX) or block.get("LoadState") == "not-found": @@ -224,14 +223,14 @@ def _loaded(blocks: List[Dict[str, str]]) -> Tuple[Dict[str, ServiceDict], Set[s return services, names -def _installed(lines: List[str], known: Set[str]) -> Dict[str, ServiceDict]: +def _installed(lines: list[str], known: set[str]) -> dict[str, ServiceDict]: """Installed services that are not loaded: neither running nor starting now. Templates, static units and aliases are left out -- the last also when an older systemd lists an alias as ``enabled``, which is why every name a loaded unit goes by is skipped. """ - services: Dict[str, ServiceDict] = {} + services: dict[str, ServiceDict] = {} for line in lines: parts = line.split() if len(parts) < 2: @@ -253,7 +252,7 @@ def _installed(lines: List[str], known: Set[str]) -> Dict[str, ServiceDict]: return services -def _apply_generated(services: Dict[str, ServiceDict], lines: List[str]) -> None: +def _apply_generated(services: dict[str, ServiceDict], lines: list[str]) -> None: """Take a generated unit's boot state from ``is-enabled``'s answer.""" for line in lines: parts = line.split() @@ -261,7 +260,7 @@ def _apply_generated(services: Dict[str, ServiceDict], lines: List[str]) -> None services[_base(parts[0])]["enabled"] = parts[1] in _ENABLED -def parse_systemd_services(output: str) -> List[ServiceDict]: +def parse_systemd_services(output: str) -> list[ServiceDict]: """Parse what :data:`SYSTEMD_SERVICES_COMMAND` printed, sorted by name. Lists every loaded service unit but those that are not found, and every @@ -299,7 +298,7 @@ class SystemdServicesMixin(ServiceControlMixin): """ ... - def get_services(self) -> List[ServiceDict]: + def get_services(self) -> list[ServiceDict]: """ Returns the services systemd knows, in one round trip. @@ -316,7 +315,7 @@ class SystemdServicesMixin(ServiceControlMixin): ) return parse_systemd_services(output) - def manage_service(self, name: str, action: str) -> Dict[str, Any]: + def manage_service(self, name: str, action: str) -> dict[str, Any]: """ Applies *action* (start, stop, restart, enable, disable) to the service *name*. diff --git a/napalm_device_types/terminal.py b/napalm_device_types/terminal.py index c5e4c00..67aa284 100644 --- a/napalm_device_types/terminal.py +++ b/napalm_device_types/terminal.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """What a pseudo-terminal adds to a command's output, taken out again. A screen-scraping transport (netmiko) gives the remote command a terminal. Tools diff --git a/napalm_device_types/tls_certificates.py b/napalm_device_types/tls_certificates.py new file mode 100644 index 0000000..6169383 --- /dev/null +++ b/napalm_device_types/tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +Network devices ship self-signed certificates no CA vouches for. A driver that +then turns verification off logs in with the device's credentials at whatever +answers on its address (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +The same answer as for SSH host keys (:mod:`napalm_device_types.ssh_host_keys`): +the layer above keeps the store of the certificates it trusts, a driver verifies. + +* It receives the fingerprints to expect in ``optional_args["tls_fingerprints"]`` + (:data:`KNOWN_FINGERPRINTS_ARG`), by the port they were seen on: + ``{"443": "SHA256:AB:CD:…"}``, the SHA-256 of the DER certificate as + ``openssl x509 -fingerprint -sha256`` prints it (the ``SHA256:`` prefix, + colons and case are optional, :func:`normalise_fingerprint`). With a + fingerprint, any other certificate is refused before a password leaves the + caller; no CA is consulted. Without one, the first certificate is accepted: + trust on first use. +* It reports every certificate it sees through + ``optional_args["tls_certificate_observer"]`` (:data:`OBSERVER_ARG`), a + callable that receives one dict per connection:: + + {"host": str, "port": int, + "seen": {"fingerprint"}, + "expected": {"fingerprint"} | None, + "accepted": bool} + + ``accepted`` is False only for a refused certificate; the connection then + raises :class:`TlsCertificateMismatch`. A failing observer is logged and + ignored. + +A caller that passes neither keeps working as before (first use, every time), +and a driver that ignores both still connects for a caller that passes them. + +:class:`CertificateCheck` is the one implementation. :meth:`~CertificateCheck.verify_server` +takes the certificate from a handshake of its own, before any request; +:meth:`~CertificateCheck.pin` makes every later connection of a ``requests`` +session present that same certificate (urllib3's ``assert_fingerprint``), so +another host cannot step in between the check and the login; +:meth:`~CertificateCheck.refusing` turns such a refusal into +:class:`TlsCertificateMismatch`; :meth:`~CertificateCheck.observe` reports the +certificate once the login stands. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import socket +import ssl +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any + +from napalm.base.exceptions import ConnectionException + +logger = logging.getLogger(__name__) + +#: The ``optional_args`` key that carries the fingerprints to expect, by port. +KNOWN_FINGERPRINTS_ARG = "tls_fingerprints" +#: The ``optional_args`` key that carries the callable every seen certificate goes to. +OBSERVER_ARG = "tls_certificate_observer" + +_PREFIX = "SHA256:" +_SHA256_HEX = re.compile(r"^[0-9A-F]{64}$") +#: What urllib3 says when a connection presents another certificate than the pinned one +#: (``got "…"``; 1.26 writes the digest as ``b'…'``, escaped once requests wraps it). +_URLLIB3_MISMATCH = re.compile(r"Fingerprints did not match.*?got \"(?:b\\?')?([0-9a-fA-F]{64})") + + +def fingerprint(der: bytes) -> str: + """``SHA256:AB:CD:…``: the SHA-256 of a DER certificate, as openssl prints it.""" + return _format(hashlib.sha256(der).hexdigest()) + + +def _format(hex_digest: str) -> str: + upper = hex_digest.upper() + return _PREFIX + ":".join(upper[i : i + 2] for i in range(0, len(upper), 2)) + + +def normalise_fingerprint(value: Any) -> str: + """*value* as :func:`fingerprint` writes it; ``SHA256:``, colons, spaces and case optional.""" + if not isinstance(value, str): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + text = value.strip() + if text[: len(_PREFIX)].upper() == _PREFIX: + text = text[len(_PREFIX) :] + digits = re.sub(r"[\s:]", "", text).upper() + if not _SHA256_HEX.match(digits): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + return _format(digits) + + +def _hex(fingerprint_: str) -> str: + """The bare lower-case hex urllib3's ``assert_fingerprint`` takes.""" + return fingerprint_.removeprefix(_PREFIX).replace(":", "").lower() + + +def fetch_certificate( + host: str, + port: int, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, +) -> bytes: + """The DER certificate *host* presents on *port*, from a handshake that sends nothing else. + + No CA and no host name are checked: the certificate is judged by its fingerprint. + """ + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with connect((host, port), timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as tls: + der = tls.getpeercert(binary_form=True) + if not der: # pragma: no cover - a TLS server always presents one + raise ConnectionException(f"{host}:{port} presented no TLS certificate") + return der + + +@dataclass(frozen=True) +class CertificateObservation: + """The certificate one connection saw, against the one it was to expect.""" + + host: str + port: int + #: The fingerprint of the certificate presented. + seen: str + #: The fingerprint it was to present; None on first use. + expected: str | None + + @property + def accepted(self) -> bool: + """True on first use and for the expected certificate; False for a refused one.""" + return self.expected is None or self.expected == self.seen + + def as_dict(self) -> dict[str, Any]: + return { + "host": self.host, + "port": self.port, + "seen": {"fingerprint": self.seen}, + "expected": {"fingerprint": self.expected} if self.expected else None, + "accepted": self.accepted, + } + + +class TlsCertificateMismatch(ConnectionException): # type: ignore[misc] + """The device presented another TLS certificate than the one it is known by. + + Raised before any request that carries credentials: nothing was sent to it. + """ + + def __init__(self, observation: CertificateObservation) -> None: + self.observation = observation + super().__init__( + f"TLS certificate mismatch at {observation.host}:{observation.port}: it presented " + f"{observation.seen}, the trusted certificate is {observation.expected or 'none'}. " + "Nothing was sent to it." + ) + + +def _expected(known: Any, port: int) -> str | None: + """The fingerprint in *known* (``{port: fingerprint}``, port as text or number) for *port*.""" + if not known: + return None + if not isinstance(known, Mapping): + raise ValueError(f"TLS fingerprints must be given by port, not as {type(known).__name__}") + value = known.get(str(port)) or known.get(port) + return normalise_fingerprint(value) if value else None + + +def _refused_fingerprint(exc: BaseException) -> str | None: + """The fingerprint urllib3 saw when it refused a pinned connection, in *exc*'s chain.""" + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + match = _URLLIB3_MISMATCH.search(str(current)) + if match: + return _format(match.group(1)) + stack.extend([current.__cause__, current.__context__]) + return None + + +class CertificateCheck: + """How one driver connection verifies its host's TLS certificate. + + *optional_args* are the driver's (both keys optional), *host* and *port* the + address the connection goes to. + """ + + def __init__(self, optional_args: Mapping[str, Any] | None, host: str, port: int) -> None: + args = optional_args or {} + self.host = host + self.port = int(port) + self.expected = _expected(args.get(KNOWN_FINGERPRINTS_ARG), self.port) + self._observer = args.get(OBSERVER_ARG) + self._verified: CertificateObservation | None = None + + def verify_server( + self, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, + ) -> CertificateObservation: + """Take the certificate from a handshake of its own and :meth:`verify` it.""" + return self.verify(fetch_certificate(self.host, self.port, timeout, connect=connect)) + + def verify(self, der: bytes) -> CertificateObservation: + """Judge the DER certificate *der*: a refused one is reported and raises. + + An accepted one is reported by :meth:`observe`, once the login stands. + """ + observation = CertificateObservation(self.host, self.port, fingerprint(der), self.expected) + if not observation.accepted: + self._report(observation) + raise TlsCertificateMismatch(observation) + self._verified = observation + return observation + + def pin(self, session: Any) -> None: + """Every HTTPS connection of the ``requests`` *session* must present the verified one.""" + verified = self._require_verified() + session.mount("https://", _pinned_adapter(_hex(verified.seen))) + + @contextmanager + def refusing(self) -> Iterator[None]: + """Around requests on a pinned session: a refused certificate becomes a mismatch.""" + try: + yield + except Exception as exc: + offered = _refused_fingerprint(exc) + if offered is None: + raise + trusted = self.expected or self._require_verified().seen + observation = CertificateObservation(self.host, self.port, offered, trusted) + self._report(observation) + raise TlsCertificateMismatch(observation) from exc + + def observe(self) -> CertificateObservation: + """After the login: the verified certificate, reported.""" + verified = self._require_verified() + self._report(verified) + return verified + + def _require_verified(self) -> CertificateObservation: + if self._verified is None: + raise RuntimeError("verify the certificate first (verify_server or verify)") + return self._verified + + def _report(self, observation: CertificateObservation) -> None: + if not callable(self._observer): + return + try: + self._observer(observation.as_dict()) + except Exception: # the store is the caller's; a connection does not hang on it + logger.warning( + "Could not report the TLS certificate of %s:%s", + self.host, + self.port, + exc_info=True, + ) + + +def _pinned_adapter(hex_digest: str) -> Any: + """A ``requests`` adapter whose connections, direct or through a proxy, must match *hex_digest*. + + ``requests`` is imported here: it comes with napalm, and a module that only + verifies a fingerprint does not need it loaded. + """ + from requests.adapters import HTTPAdapter + + class _PinnedAdapter(HTTPAdapter): # type: ignore[misc,unused-ignore] + def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: + kwargs["assert_fingerprint"] = hex_digest + super().init_poolmanager(*args, **kwargs) + + def proxy_manager_for(self, proxy: str, **kwargs: Any) -> Any: + kwargs["assert_fingerprint"] = hex_digest + return super().proxy_manager_for(proxy, **kwargs) + + return _PinnedAdapter() diff --git a/napalm_device_types/updates.py b/napalm_device_types/updates.py index 56e7934..7660181 100644 --- a/napalm_device_types/updates.py +++ b/napalm_device_types/updates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Pending package updates and applying them. The two device types that offer this used to declare it under two different @@ -13,15 +12,15 @@ this class in can never shadow a working implementation from a sibling base. from __future__ import annotations -from typing import Any, Dict, List, TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict +from napalm_device_types.models import ApplyUpdatesResultDict, UpdateDict, UpdateProgressDict class UpdateMixin: if TYPE_CHECKING: - def get_available_updates(self) -> List[UpdateDict]: + def get_available_updates(self) -> list[UpdateDict]: """ Returns a list of packages that have a newer version available. @@ -51,7 +50,7 @@ class UpdateMixin: """ ... - def refresh_available_updates(self) -> Dict[str, Any]: + def refresh_available_updates(self) -> dict[str, Any]: """ Refreshes the host's package index, so that :meth:`get_available_updates` reports what the repositories offer now (``apt-get update``, @@ -61,7 +60,19 @@ class UpdateMixin: """ ... - def apply_updates(self, packages: List[str]) -> ApplyUpdatesResultDict: + def get_update_progress(self) -> UpdateProgressDict: + """ + Where an update the device runs by itself stands, for a caller that + follows it: a firewall's firmware upgrade, which :meth:`apply_updates` + only starts. ``state`` is ``"running"`` until it ends in ``"done"`` + or ``"failed"``; ``log`` is the device's whole log of it. + + Declared only: a driver whose :meth:`apply_updates` returns when the + update is over has nothing to report here. + """ + ... + + def apply_updates(self, packages: list[str]) -> ApplyUpdatesResultDict: """ Upgrades the given packages to the newest available version. diff --git a/napalm_device_types/virtual_ips.py b/napalm_device_types/virtual_ips.py index 6733684..a8bb46b 100644 --- a/napalm_device_types/virtual_ips.py +++ b/napalm_device_types/virtual_ips.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Virtual IPs: the addresses a host's HA configuration lets float between machines. vip-manager and keepalived put a service address on whichever member is master @@ -51,9 +50,10 @@ from __future__ import annotations import ipaddress import re +from collections.abc import Callable from dataclasses import dataclass, field from shlex import quote -from typing import Callable, Dict, List, Optional, Set, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING from napalm_device_types.models import VirtualIpDict, VirtualIpsDict from napalm_device_types.terminal import strip_terminal_codes @@ -108,7 +108,7 @@ _AWK = " ".join( r" if (!ok(f)) return; seen[f] = 1;" r' while (0 < (r = (getline l < f))) { sub(/[#!].*/, "", l); gsub(/[{}]/, " & ", l);' r' n = split(l, t, " "); if (n && nw && t[1] != "{") { stmt(w, nw, f, d); nw = 0 }' - r' for (k = 1; k <= n; k++) {' + r" for (k = 1; k <= n; k++) {" r' if (t[k] == "{") { push(nw ? w[1] : "", 1 < nw ? w[2] : ""); nw = 0 }' r' else if (t[k] == "}") { stmt(w, nw, f, d); nw = 0; pop() } else w[++nw] = t[k] } }' r" stmt(w, nw, f, d);" @@ -157,10 +157,10 @@ VIRTUAL_IPS_COMMAND = ( _COMMENT_RE = re.compile(r"\s+#.*$") _VARIABLE_RE = re.compile(r"^\$\{?(\w+)\}?$") -_Record = Tuple[str, str] +_Record = tuple[str, str] -def _frame(output: str) -> List[str]: +def _frame(output: str) -> list[str]: lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] try: start = lines.index(_BEGIN) @@ -170,9 +170,9 @@ def _frame(output: str) -> List[str]: return lines[start + 1 : end] -def _sections(lines: List[str]) -> List[Tuple[str, str, List[_Record]]]: +def _sections(lines: list[str]) -> list[tuple[str, str, list[_Record]]]: """``[ ]`` headers, each with its records as ``(kind, rest)``.""" - sections: List[Tuple[str, str, List[_Record]]] = [] + sections: list[tuple[str, str, list[_Record]]] = [] for line in lines: if line.startswith("[") and line.endswith("]"): mechanism, _, unit = line[1:-1].partition(" ") @@ -191,7 +191,7 @@ def _clean(value: str) -> str: return value.strip() -def _address(text: str) -> Optional[Tuple[str, Optional[int]]]: +def _address(text: str) -> tuple[str, int | None] | None: """``"10.0.0.10/24"`` -> ``("10.0.0.10", 24)``; None for what is no address.""" host, slash, prefix = text.partition("/") try: @@ -205,7 +205,7 @@ def _address(text: str) -> Optional[Tuple[str, Optional[int]]]: return str(address), int(prefix) -def _prefix(mask: Optional[str]) -> Optional[int]: +def _prefix(mask: str | None) -> int | None: """``"24"`` or ``"255.255.255.0"`` -> 24.""" if not mask: return None @@ -217,27 +217,27 @@ def _prefix(mask: Optional[str]) -> Optional[int]: return None -def _running(records: List[_Record]) -> Optional[bool]: +def _running(records: list[_Record]) -> bool | None: states = [rest for kind, rest in records if kind == "active"] return states[-1] == "active" if states else None -def _vip_manager(unit: str, records: List[_Record]) -> List[VirtualIpDict]: - values: Dict[str, Dict[str, str]] = {"arg": {}, "env": {}, "cfg": {}} +def _vip_manager(unit: str, records: list[_Record]) -> list[VirtualIpDict]: + values: dict[str, dict[str, str]] = {"arg": {}, "env": {}, "cfg": {}} for kind, rest in records: if kind in values: key, _, value = rest.partition(" ") values[kind][key] = value env = values["env"] - def resolve(value: str) -> Optional[str]: + def resolve(value: str) -> str | None: value = _clean(value) variable = _VARIABLE_RE.match(value) if variable: value = _clean(env.get(variable.group(1), "")) return value if value and "$" not in value else None - def pick(args: Tuple[str, ...], envs: Tuple[str, ...], cfgs: Tuple[str, ...]) -> Optional[str]: + def pick(args: tuple[str, ...], envs: tuple[str, ...], cfgs: tuple[str, ...]) -> str | None: for kind, names in (("arg", args), ("env", envs), ("cfg", cfgs)): for name in names: if name in values[kind]: @@ -272,15 +272,15 @@ def _vip_manager(unit: str, records: List[_Record]) -> List[VirtualIpDict]: @dataclass class _Instance: name: str - interface: Optional[str] = None + interface: str | None = None vrid: str = "" - vips: List[str] = field(default_factory=list) + vips: list[str] = field(default_factory=list) -def _keepalived(unit: str, records: List[_Record]) -> List[VirtualIpDict]: +def _keepalived(unit: str, records: list[_Record]) -> list[VirtualIpDict]: running = _running(records) - instances: List[_Instance] = [] - current: Optional[_Instance] = None + instances: list[_Instance] = [] + current: _Instance | None = None for kind, rest in records: if kind == "instance": current = _Instance(rest or unit) @@ -298,8 +298,8 @@ def _keepalived(unit: str, records: List[_Record]) -> List[VirtualIpDict]: return [entry for instance in instances for entry in _instance_entries(instance, running)] -def _instance_entries(instance: _Instance, running: Optional[bool]) -> List[VirtualIpDict]: - entries: List[VirtualIpDict] = [] +def _instance_entries(instance: _Instance, running: bool | None) -> list[VirtualIpDict]: + entries: list[VirtualIpDict] = [] for vip in instance.vips: tokens = vip.split() parsed = _address(tokens[0]) if tokens else None @@ -326,13 +326,13 @@ def _host_prefix(address: str) -> int: return ipaddress.ip_address(address).max_prefixlen -_READERS: Dict[str, Tuple[str, Callable[[str, List[_Record]], List[VirtualIpDict]]]] = { +_READERS: dict[str, tuple[str, Callable[[str, list[_Record]], list[VirtualIpDict]]]] = { "vip-manager": ("vip_manager", _vip_manager), "keepalived": ("keepalived", _keepalived), } -def _order(entry: VirtualIpDict) -> Tuple[int, int, str]: +def _order(entry: VirtualIpDict) -> tuple[int, int, str]: address = ipaddress.ip_address(entry["address"]) return address.version, int(address), entry["instance"] @@ -346,8 +346,8 @@ def parse_virtual_ips(output: str) -> VirtualIpsDict: sections = _sections(_frame(output)) if any(mechanism == _NO_SYSTEMD for mechanism, _, _ in sections): return {"vip_manager": None, "keepalived": None} - found: Dict[str, List[VirtualIpDict]] = {"vip_manager": [], "keepalived": []} - blind: Set[str] = set() + found: dict[str, list[VirtualIpDict]] = {"vip_manager": [], "keepalived": []} + blind: set[str] = set() for mechanism, unit, records in sections: if mechanism not in _READERS: continue @@ -356,7 +356,8 @@ def parse_virtual_ips(output: str) -> VirtualIpsDict: blind.add(key) else: found[key].extend(read(unit, records)) - def result(key: str) -> Optional[List[VirtualIpDict]]: + + def result(key: str) -> list[VirtualIpDict] | None: return None if key in blind else sorted(found[key], key=_order) return {"vip_manager": result("vip_manager"), "keepalived": result("keepalived")} diff --git a/napalm_device_types/vm_migration.py b/napalm_device_types/vm_migration.py new file mode 100644 index 0000000..23d14c2 --- /dev/null +++ b/napalm_device_types/vm_migration.py @@ -0,0 +1,26 @@ +"""Moving a guest to another node of its cluster (``HypervisorDriver.migrate_vm``). + +``get_vm_migration_preflight`` reports what the platform says before a move: +which nodes would take the guest, which refuse it and why, and what the guest +holds on its current node. ``migrate_vm`` then moves it and blocks until the +platform's task ended; a failure raises with the platform's own cause. +``set_node_maintenance`` switches a node's HA maintenance mode, which makes the +HA manager move its HA guests away while it is on. + +Facts only: which node to pick, and whether a local disk is worth copying, is +the caller's to decide. +""" + +from typing import TypedDict + + +class MigrationPreflightDict(TypedDict): + """Where a guest could move to, and what stands in the way.""" + + running: bool + allowed_nodes: list[str] # nodes the platform would move it to + refused_nodes: dict[str, str] # node -> the platform's reason, in plain words + # volume ids on storage the target does not share; they block nothing by + # themselves, the migration copies them + local_disks: list[str] + local_resources: list[str] # e.g. "hostpci0", "usb0": they block the migration diff --git a/napalm_device_types/vm_stop_request.py b/napalm_device_types/vm_stop_request.py new file mode 100644 index 0000000..5d0b5de --- /dev/null +++ b/napalm_device_types/vm_stop_request.py @@ -0,0 +1,29 @@ +"""Asking for a VM's clean shutdown without waiting for it. + +``HypervisorDriver.stop_vm`` blocks until the VM is stopped, within a limit +of the driver's own. A caller that keeps its waits as timers asks with +``begin_stop_vm`` and reads ``get_vm_status`` until the VM says it stopped. + +Declared under ``if TYPE_CHECKING``: a driver mixes this in when its +hypervisor can take the request on its own; ``hasattr`` tells a caller +whether it did. No role base carries it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + + +class VmStopRequestMixin: + if TYPE_CHECKING: + + def begin_stop_vm(self, name: str) -> None: + """ + Asks the hypervisor to shut a virtual machine down cleanly (ACPI) + and returns as soon as the request was accepted. + + :param name: VM name or its ``vmid``. + :raises ValueError: If no VM with the given name/ID exists. + :raises RuntimeError: If the hypervisor refuses the request. + """ + ... diff --git a/pyproject.toml b/pyproject.toml index 2b541c9..0039557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,14 @@ [build-system] -requires = ["setuptools>=68", "wheel"] +requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.9.0" +version = "4.23.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" -requires-python = ">=3.10" -license = { text = "Apache-2.0" } +requires-python = ">=3.12" +license = "Apache-2.0" authors = [ { name = "Christian Manivong", email = "christian@manivong.de" }, ] @@ -28,11 +28,8 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Intended Audience :: System Administrators", - "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: System :: Networking", @@ -40,21 +37,22 @@ classifiers = [ "Typing :: Typed", ] dependencies = [ - "napalm>=4.0", + "napalm>=5.1", # older releases import pkg_resources, which a 3.12 venv lacks + "requests>=2.25.0", "typing-extensions>=4.0", ] [project.urls] -Homepage = "https://github.com/chrismanivong/napalm-device-types" -Repository = "https://github.com/chrismanivong/napalm-device-types" -Issues = "https://github.com/chrismanivong/napalm-device-types/issues" +Repository = "https://git.netork.io/NAPALM/napalm-device-types" +Issues = "https://git.netork.io/NAPALM/napalm-device-types/issues" +Changelog = "https://git.netork.io/NAPALM/napalm-device-types/src/branch/main/CHANGELOG.md" [project.optional-dependencies] dev = [ - "pytest", - "mypy", - "build", - "twine", + "pytest>=8", + "pytest-cov>=5", + "mypy>=1.11", + "ruff>=0.6", ] [tool.setuptools.packages.find] @@ -64,7 +62,27 @@ include = ["napalm_device_types*"] [tool.setuptools.package-data] napalm_device_types = ["py.typed"] +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] + +[tool.ruff.lint.per-file-ignores] +# Recorded device output keeps its lines. +"tests/**" = ["E501"] + [tool.mypy] -python_version = "3.10" +python_version = "3.12" strict = true ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.run] +source = ["napalm_device_types"] + +[tool.coverage.report] +fail_under = 95 diff --git a/tests/quality_baseline.json b/tests/quality_baseline.json new file mode 100644 index 0000000..3b89766 --- /dev/null +++ b/tests/quality_baseline.json @@ -0,0 +1,38 @@ +{ + "coverage_floor": 95.0, + "function_complexity": { + "napalm_device_types/_ucd_metrics.py::build_if_metrics": 20, + "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 17, + "napalm_device_types/host_firewall.py::_iptables_rule": 11, + "napalm_device_types/host_firewall.py::_module_match": 13, + "napalm_device_types/host_firewall.py::_module_options": 14, + "napalm_device_types/host_firewall.py::_nft_field": 15, + "napalm_device_types/host_firewall.py::_nft_match": 12, + "napalm_device_types/host_firewall.py::_nft_rule": 16, + "napalm_device_types/host_firewall.py::_parse_nft": 14, + "napalm_device_types/host_firewall.py::rule_text": 14, + "napalm_device_types/host_status.py::_reboot": 12, + "napalm_device_types/lag.py::add_lag_interfaces": 11, + "napalm_device_types/listening.py::parse_listening_sockets": 13, + "napalm_device_types/service_clusters.py::_patroni": 11, + "napalm_device_types/ssh_server.py::plan_ssh_hardening": 38, + "napalm_device_types/virtual_ips.py::_keepalived": 12 + }, + "function_lines": { + "napalm_device_types/_ucd_metrics.py::collect_ucd_metrics": 119, + "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_reservations": 54, + "napalm_device_types/dhcp.py::DhcpServerMixin.diff_dhcp_subnets": 60, + "napalm_device_types/firewall_rules.py::FirewallRuleMixin.diff_firewall_rules": 56, + "napalm_device_types/hypervisor.py::HypervisorDriver.create_vm_from_cloud_init": 84, + "napalm_device_types/hypervisor.py::HypervisorDriver.get_vm_config": 77, + "napalm_device_types/ssh_server.py::plan_ssh_hardening": 60, + "napalm_device_types/storage.py::StorageDriver.get_volumes": 52, + "napalm_device_types/switch.py::SwitchDriver.get_dot1x_ports": 51, + "napalm_device_types/switch.py::SwitchDriver.get_poe_status": 63 + }, + "mypy_relaxed": [], + "python_file_lines": { + "napalm_device_types/models.py": 1463 + }, + "untyped_public": [] +} diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py new file mode 100644 index 0000000..b886211 --- /dev/null +++ b/tests/test_apt_proxy.py @@ -0,0 +1,826 @@ +"""apt's proxy: what apt fetches through, whether a proxy answers, pointing apt at one. + +apt reads every file in ``apt.conf.d`` in order and ``apt.conf`` last; the last +value wins. The Debian installer writes its proxy into ``apt.conf``, Ubuntu's +into ``apt.conf.d/90curtin-aptproxy``: reading one file was not enough +(NetOrk/netork#1397). The fetch outputs are apt-helper's own, from a Mint 22.3 +host against netOrk's test server. +""" + +from __future__ import annotations + +import os +import re +import subprocess + +import pytest + +from napalm_device_types.apt_proxy import ( + APT_PROXY_COMMAND, + AptProxyMixin, + apt_fetch_command, + comment_out_command, + https_proxy_for, + parse_apt_fetch, + parse_apt_proxy, + same_proxy_url, +) +from napalm_device_types.fixes import PackageProxyMixin + + +def _report(dump=(), parts=(), main=(), sources=()): + lines = ["APROXY_BEGIN", "[dump]", *dump, "[parts]", *parts, "[main]", *main] + return "\n".join([*lines, "[sources]", *sources, "APROXY_END"]) + + +_D = "/etc/apt/apt.conf.d/" + + +class TestEffectiveProxy: + def test_the_proxy_apt_fetches_through(self): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://c:3142";'], + parts=[f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";'], + ) + ) + + assert report["proxy"] == "http://c:3142" + assert report["settings"] == [ + {"file": f"{_D}00proxy", "line": 1, "value": "http://c:3142", "editable": True} + ] + + def test_none_without_a_proxy(self): + report = parse_apt_proxy(_report()) + + assert report["proxy"] is None + assert report["settings"] == [] + assert report["per_host"] == {} + assert report["auto_detect"] is None + + @pytest.mark.parametrize("value", ["", "DIRECT", "direct"]) + def test_an_empty_or_direct_proxy_is_none(self, value): + assert parse_apt_proxy(_report(dump=[f'Acquire::http::Proxy "{value}";']))["proxy"] is None + + def test_keys_are_case_insensitive(self): + report = parse_apt_proxy(_report(dump=['acquire::HTTP::proxy "http://c:3142";'])) + assert report["proxy"] == "http://c:3142" + + def test_per_host_overrides_and_auto_detect(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::http::Proxy::deb.debian.org "DIRECT";', + 'Acquire::http::Proxy::ppa.launchpad.net "http://other:3128";', + 'Acquire::http::Proxy-Auto-Detect "/usr/bin/auto-apt-proxy";', + ] + ) + ) + + assert report["proxy"] == "http://c:3142" + assert report["per_host"] == { + "deb.debian.org": "DIRECT", + "ppa.launchpad.net": "http://other:3128", + } + assert report["auto_detect"] == "/usr/bin/auto-apt-proxy" + + def test_no_intact_report_raises(self): + with pytest.raises(ValueError): + parse_apt_proxy("APROXY_BEGIN\n[dump]\n") + + +class TestWhereItIsSet: + def test_every_setting_in_apt_s_reading_order(self): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://old:3128";'], + parts=[ + f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";', + f'{_D}90curtin-aptproxy:1:Acquire::http::Proxy "http://old:3128";', + ], + main=['/etc/apt/apt.conf:4: Acquire::http::Proxy "http://older:8080";'], + ) + ) + + assert [(s["file"], s["line"], s["value"]) for s in report["settings"]] == [ + (f"{_D}00proxy", 1, "http://c:3142"), + (f"{_D}90curtin-aptproxy", 1, "http://old:3128"), + ("/etc/apt/apt.conf", 4, "http://older:8080"), + ] + + def test_a_nested_setting_is_named_but_not_editable(self): + report = parse_apt_proxy(_report(parts=[f'{_D}01proxy:2: Proxy "http://nested:3142";'])) + + assert report["settings"] == [ + {"file": f"{_D}01proxy", "line": 2, "value": "http://nested:3142", "editable": False} + ] + + @pytest.mark.parametrize( + "line", + [ + '// Acquire::http::Proxy "http://c:3142";', + '# Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy "http://c:3142";', + 'Acquire::ftp::Proxy "http://c:3142";', + 'Acquire::http::Proxy::deb.debian.org "DIRECT";', + 'Acquire::http::Proxy-Auto-Detect "/usr/bin/x";', + "Acquire::http {", + ], + ) + def test_what_does_not_set_the_http_proxy_is_left_out(self, line): + assert parse_apt_proxy(_report(parts=[f"{_D}00x:1:{line}"]))["settings"] == [] + + @pytest.mark.parametrize("name", ["00proxy.bak", "00proxy.dpkg-old", "00proxy.save", "a b"]) + def test_files_apt_does_not_read_are_left_out(self, name): + line = f'{_D}{name}:1:Acquire::http::Proxy "http://c:3142";' + assert parse_apt_proxy(_report(parts=[line]))["settings"] == [] + + def test_a_conf_extension_is_read(self): + line = f'{_D}01proxy.conf:1:Acquire::http::Proxy "http://c:3142";' + assert len(parse_apt_proxy(_report(parts=[line]))["settings"]) == 1 + + +class TestSources: + def test_sources_by_scheme(self): + report = parse_apt_proxy( + _report( + sources=[ + "http://archive.ubuntu.com/ubuntu/dists/noble/", + "https://download.docker.com/linux/ubuntu/dists/noble/", + ] + ) + ) + + assert report["http_sources"] == ["http://archive.ubuntu.com/ubuntu/dists/noble/"] + assert report["https_sources"] == ["https://download.docker.com/linux/ubuntu/dists/noble/"] + + +class TestHttpsProxy: + """apt's https method asks ``Acquire::https::Proxy`` and then ``Acquire::http::Proxy``: + an http proxy alone takes the https sources too (NetOrk/netork#1811). Measured with + apt 2.8.3's apt-helper against a dead port.""" + + def test_the_http_proxy_alone_takes_https_too(self): + report = parse_apt_proxy(_report(dump=['Acquire::http::Proxy "http://c:3142";'])) + assert report["https_proxy"] == "http://c:3142" + + def test_an_https_proxy_of_its_own_wins(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy "http://other:3128";', + ] + ) + ) + + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", "http://other:3128") + + @pytest.mark.parametrize("value", ["DIRECT", "direct"]) + def test_direct_https_is_none(self, value): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://c:3142";', f'Acquire::https::Proxy "{value}";'] + ) + ) + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", None) + + def test_an_empty_https_proxy_is_not_set(self): + """apt takes an empty value for unset and goes on to the http one.""" + report = parse_apt_proxy( + _report(dump=['Acquire::http::Proxy "http://c:3142";', 'Acquire::https::Proxy "";']) + ) + assert report["https_proxy"] == "http://c:3142" + + def test_per_host_https_then_http(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy::a.example "DIRECT";', + 'Acquire::http::Proxy::b.example "DIRECT";', + 'Acquire::https::Proxy::b.example "http://c:3142";', + ] + ) + ) + + assert report["https_per_host"] == {"a.example": "DIRECT", "b.example": "http://c:3142"} + assert report["per_host"] == {"a.example": "DIRECT", "b.example": "DIRECT"} + + def test_where_the_https_proxy_is_set(self): + report = parse_apt_proxy( + _report( + parts=[ + f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";', + f'{_D}90curtin-aptproxy:2:Acquire::https::Proxy "http://c:3142";', + f'{_D}91x:1:Acquire::https {{ Proxy "http://n:1"; }};', + f'{_D}92x:1:// Acquire::https::Proxy "http://c:3142";', + f'{_D}93x:1:Acquire::https::Proxy::h.example "DIRECT";', + ] + ) + ) + + assert report["https_settings"] == [ + { + "file": f"{_D}90curtin-aptproxy", + "line": 2, + "value": "http://c:3142", + "editable": True, + }, + {"file": f"{_D}91x", "line": 1, "value": "http://n:1", "editable": False}, + ] + assert [s["file"] for s in report["settings"]] == [f"{_D}00proxy"] + + @pytest.mark.parametrize( + "host, proxy", + [ + ("deb.example", "http://c:3142"), + ("direct.example", None), + ("Other.Example", "http://o:1"), + ], + ) + def test_the_proxy_for_one_host(self, host, proxy): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy::direct.example "DIRECT";', + 'Acquire::http::Proxy::other.example "http://o:1";', + ] + ) + ) + assert https_proxy_for(report, host) == proxy + + +class TestSameProxyUrl: + @pytest.mark.parametrize( + "a, b", + [ + ("http://c:3142", "http://c:3142/"), + ("http://C:3142", "http://c:3142"), + ("http://c", "http://c:80"), + ], + ) + def test_the_same(self, a, b): + assert same_proxy_url(a, b) + + @pytest.mark.parametrize( + "a, b", + [ + ("http://c:3142", "http://c:31420"), + ("http://c:3142", "http://c:3142.evil"), + ("http://c:3142", "https://c:3142"), + ("http://c:3142", ""), + ("http://c:3142", None), + ], + ) + def test_not_the_same(self, a, b): + assert not same_proxy_url(a, b) + + +_REFUSED = """Err:1 http://172.22.8.50:3143/acng-report.html + Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused) +E: Failed to fetch http://172.22.8.50:3143/acng-report.html Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused) +E: Download Failed""" + +_HTTP_ERROR = """Err:1 http://172.22.8.50:3142/nothing-here.html + 503 Host not found [IP: 172.22.8.50 3142] +E: Failed to fetch http://172.22.8.50:3142/nothing-here.html 503 Host not found [IP: 172.22.8.50 3142] +E: Download Failed""" + +_UNRESOLVED = """Err:1 http://no-such-host.invalid:3142/acng-report.html + Could not resolve 'no-such-host.invalid' +E: Failed to fetch http://no-such-host.invalid:3142/acng-report.html Could not resolve 'no-such-host.invalid' +E: Download Failed""" + +_FETCHED = """Get:1 http://172.22.8.50:3142/acng-report.html +Fetched 14.0 kB in 0s (0 B/s)""" + +#: apt-cacher-ng without a PassThroughPattern, as apt 2.8.3 tells it (NetOrk/netork#1811). +_CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/InRelease + Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Download Failed""" + + +def _fetch(output, rc, match=None): + lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] + if match is not None: + lines += ["[match]", str(match)] + return "\n".join([*lines, "APROXY_END"]) + + +class TestFetch: + def test_fetched_and_expected(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0, match=4)) == { + "reachable": True, + "expected": True, + "error": "", + } + + def test_fetched_but_not_the_expected_page(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0, match=0))["expected"] is False + + def test_without_an_expectation_expected_is_none(self): + assert parse_apt_fetch(_fetch(_FETCHED, 0))["expected"] is None + + def test_refused(self): + result = parse_apt_fetch(_fetch(_REFUSED, 100)) + + assert result["reachable"] is False + assert result["error"] == ( + "Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused)" + ) + + def test_unresolved(self): + result = parse_apt_fetch(_fetch(_UNRESOLVED, 100)) + assert (result["reachable"], result["error"]) == ( + False, + "Could not resolve 'no-such-host.invalid'", + ) + + def test_an_http_error_was_an_answer(self): + result = parse_apt_fetch(_fetch(_HTTP_ERROR, 100)) + + assert result["reachable"] is True + assert result["expected"] is False + assert result["error"] == "503 Host not found [IP: 172.22.8.50 3142]" + + def test_a_refused_tunnel(self): + result = parse_apt_fetch(_fetch(_CONNECT_DENIED, 100)) + + assert result["reachable"] is False + assert "403 CONNECT denied" in result["error"] + + def test_without_apt_helper_nothing_could_be_tried(self): + with pytest.raises(ValueError, match="apt-helper"): + parse_apt_fetch(_fetch("sh: 1: /usr/lib/apt/apt-helper: not found", 127)) + + def test_no_intact_report_raises(self): + with pytest.raises(ValueError): + parse_apt_fetch("APROXY_BEGIN\n[fetch]\n") + + +class TestFetchCommand: + def test_past_the_proxy_to_the_url_itself(self): + command = apt_fetch_command( + "http://c:3142/acng-report.html", direct=True, expect="Cache NG" + ) + + assert "'Acquire::http::Proxy::c=DIRECT'" in command + assert "grep -c -F -- 'Cache NG'" in command + + def test_through_apt_s_own_proxy(self): + command = apt_fetch_command("http://archive.ubuntu.com/ubuntu/dists/noble/InRelease") + assert "DIRECT" not in command + + def test_an_https_source_through_apt_s_own_proxy(self): + command = apt_fetch_command( + "https://download.docker.com/linux/ubuntu/dists/noble/InRelease" + ) + assert "DIRECT" not in command + assert "download-file https://download.docker.com/" in command + + def test_past_the_proxy_to_an_https_host(self): + """For https the host's own https setting comes first; the http one alone + would lose to an https proxy for that host.""" + command = apt_fetch_command("https://h.example/x", direct=True) + + assert "'Acquire::https::Proxy::h.example=DIRECT'" in command + assert "'Acquire::http::Proxy::h.example=DIRECT'" in command + + @pytest.mark.parametrize("url", ["http://c:3142/x'; rm -rf /", "ftp://c/x", "c:3142"]) + def test_a_url_that_would_break_the_command_is_refused(self, url): + with pytest.raises(ValueError): + apt_fetch_command(url) + + +def test_the_commands_are_valid_posix_sh(): + for command in ( + APT_PROXY_COMMAND, + apt_fetch_command("http://c:3142/x", direct=True, expect="y"), + apt_fetch_command("https://h.example/dists/x/InRelease"), + ): + subprocess.run(["sh", "-n", "-c", command], check=True) + + +@pytest.mark.skipif(not os.path.exists("/usr/bin/apt-config"), reason="needs an apt host") +def test_the_report_command_on_this_host(): + output = subprocess.run(["sh", "-c", APT_PROXY_COMMAND], capture_output=True, text=True).stdout + report = parse_apt_proxy(output) + assert set(report) >= {"proxy", "https_proxy", "settings", "https_settings", "https_sources"} + + +_DOCKER = "https://download.docker.com/linux/ubuntu/dists/noble/" + + +class _Host(AptProxyMixin): + """A Debian host: its apt files, and what apt-config would make of them. + + *https_sources* are its sources over https. *tunnel* is whether the proxy lets + them through; *direct_rc* is how a fetch of one goes that takes no proxy. + """ + + def __init__( + self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0 + ): + self.files = dict(files or {}) + self.fetch_rc = fetch_rc + self.apt = apt + self.https_sources = list(https_sources) + self.tunnel = tunnel + self.direct_rc = direct_rc + self.root = [] + self.fetched = [] + + def _is_apt_host(self): + return self.apt + + def _settings(self): + out = [] + for path in sorted(p for p in self.files if p.startswith(_D)) + [ + p for p in self.files if p == "/etc/apt/apt.conf" + ]: + for n, text in enumerate(self.files[path].splitlines(), 1): + if "proxy" in text.lower(): + out.append((path, n, text)) + return out + + def _effective(self, scheme="http"): + value = None + for _path, _n, text in self._settings(): + m = re.match(rf'\s*Acquire::{scheme}::Proxy\s+"([^"]*)"', text, re.I) + if m: + value = m.group(1) + return value + + def _run_apt_command(self, command): + if command == APT_PROXY_COMMAND: + value, https = self._effective(), self._effective("https") + dump = [f'Acquire::http::Proxy "{value}";'] if value else [] + dump += [f'Acquire::https::Proxy "{https}";'] if https else [] + return _report( + dump=dump, + parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], + main=[f"{p}:{n}:{t}" for p, n, t in self._settings() if not p.startswith(_D)], + sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources], + ) + self.fetched.append(command) + if re.search(r"download-file '?https://", command): + return self._fetch_https(command) + return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) + + def _fetch_https(self, command): + https = self._effective("https") or self._effective() + if "DIRECT" in command or not https or https.upper() == "DIRECT": + return _fetch(_FETCHED if self.direct_rc == 0 else _UNRESOLVED, self.direct_rc) + return _fetch(_FETCHED, 0) if self.tunnel else _fetch(_CONNECT_DENIED, 100) + + def _write_apt_file(self, path, content): + self.files[path] = content + return {"ok": True, "output": f"wrote {path}"} + + def _run_apt_root_command(self, command, done): + self.root.append(command) + path = command.rsplit(" ", 1)[-1].strip("'") + for m in re.finditer(r"(\d+)\{/[^}]*\}", command): + n = int(m.group(1)) + lines = self.files[path].splitlines() + lines[n - 1] = "// " + lines[n - 1] + self.files[path] = "\n".join(lines) + "\n" + if command.startswith("rm -f "): + self.files.pop(command.split()[-1].strip("'"), None) + return {"ok": True, "output": done} + + +class TestCheckHttpsSource: + def test_one_source_the_way_apt_fetches_it(self): + host = _Host( + {f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.check_https_source() + + assert result["url"] == f"{_DOCKER}InRelease" + assert result["proxy"] == "http://c:3142" + assert result["reachable"] is False and "403 CONNECT denied" in result["error"] + assert "DIRECT" not in host.fetched[0] + + def test_none_without_an_https_source(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + assert host.check_https_source() is None + + def test_none_where_https_goes_direct(self): + host = _Host( + { + f"{_D}00proxy": ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + }, + https_sources=[_DOCKER], + ) + assert host.check_https_source() is None + + def test_not_on_a_host_without_apt(self): + with pytest.raises(NotImplementedError): + _Host(apt=False).check_https_source() + + +class TestTheMixin: + def test_it_is_a_package_proxy(self): + assert issubclass(AptProxyMixin, PackageProxyMixin) + + def test_the_fact(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + + assert host.get_package_proxy() == "http://c:3142" + assert host.get_package_proxy_report()["settings"][0]["file"] == f"{_D}00proxy" + + def test_not_an_apt_host(self): + host = _Host(apt=False) + + assert host.get_package_proxy() is None + assert host.get_package_proxy_report() is None + + def test_check_fetches_the_page_past_any_proxy(self): + host = _Host() + + result = host.check_package_proxy("http://c:3142/acng-report.html", expect="Cache") + + assert result == {"reachable": True, "expected": True, "error": ""} + assert "Proxy::c=DIRECT" in host.fetched[0] + + +def _step(result, name): + return next(s for s in result["steps"] if s["step"] == name) + + +class TestConfigurePackageProxy: + def test_a_fresh_host(self): + host = _Host() + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert [s["step"] for s in result["steps"]] == [ + "configure", + "conflicts", + "verify", + "fetch", + "https", + ] + assert "no https source" in result["steps"][-1]["output"] + assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' + assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] + + def test_a_later_file_is_commented_out_and_named(self): + host = _Host( + { + f"{_D}90curtin-aptproxy": 'Acquire::http::Proxy "http://old:3128";\n', + "/etc/apt/apt.conf": 'APT::Foo "1";\nAcquire::http::Proxy "http://older:8080";\n', + } + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}90curtin-aptproxy"].startswith("// Acquire::http::Proxy") + assert host.files["/etc/apt/apt.conf"] == ( + 'APT::Foo "1";\n// Acquire::http::Proxy "http://older:8080";\n' + ) + conflicts = result["steps"][1]["output"] + assert f"{_D}90curtin-aptproxy:1 (http://old:3128)" in conflicts + assert "/etc/apt/apt.conf:2 (http://older:8080)" in conflicts + + def test_the_same_proxy_elsewhere_stays(self): + host = _Host({f"{_D}01proxy": 'Acquire::http::Proxy "http://c:3142/";\n'}) + + host.configure_package_proxy("http://c:3142") + + assert host.root == [] + + def test_a_setting_it_cannot_comment_out_fails_verification(self): + host = _Host({"/etc/apt/apt.conf": 'Acquire::http { Proxy "http://older:8080"; };\n'}) + host._effective = lambda scheme="http": "http://older:8080" + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "verify" + assert "http://older:8080" in result["steps"][-1]["output"] + assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"] + + def test_a_dead_proxy_fails_the_fetch(self): + host = _Host(fetch_rc=100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "fetch" + assert "Connection refused" in result["steps"][-1]["output"] + + def test_an_https_source_the_cache_tunnels_stays_with_it(self): + host = _Host(https_sources=[_DOCKER]) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' + assert result["steps"][-1]["output"] == ( + f"fetched {_DOCKER}InRelease through http://c:3142" + ) + + def test_an_https_source_the_cache_refuses_goes_direct(self): + """NetOrk/netork#1811: apt-cacher-ng without a PassThroughPattern answers + 403 to every https source apt sends it.""" + host = _Host(https_sources=[_DOCKER], tunnel=False) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + output = result["steps"][-1]["output"] + assert "http://c:3142 refused https" in output and "403 CONNECT denied" in output + assert f"https sources go direct now: fetched {_DOCKER}InRelease" in output + + def test_a_later_https_line_to_the_cache_is_commented_out(self): + """Ubuntu's installer writes the https proxy too; it would win over 00proxy.""" + curtin = f"{_D}90curtin-aptproxy" + host = _Host( + { + curtin: ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Acquire::https::Proxy "http://c:3142";\n' + ) + }, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[curtin] == ( + 'Acquire::http::Proxy "http://c:3142";\n// Acquire::https::Proxy "http://c:3142";\n' + ) + assert f"{curtin}:2 (http://c:3142) commented out" in result["steps"][-1]["output"] + + def test_an_https_proxy_of_its_own_is_not_the_fix_s(self): + host = _Host( + {f"{_D}01https": 'Acquire::https::Proxy "http://corp:3128";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "no https source goes through http://c:3142" in result["steps"][-1]["output"] + assert f"{_D}00proxy" in host.files and "https" not in host.files[f"{_D}00proxy"] + + def test_an_http_error_through_the_tunnel_is_a_tunnel(self): + """The mirror answered, so the cache let the source through.""" + host = _Host(https_sources=[_DOCKER]) + host._fetch_https = lambda command: _fetch(_HTTP_ERROR, 100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "https" not in host.files[f"{_D}00proxy"] + + def test_refused_and_no_way_direct_fails(self): + host = _Host(https_sources=[_DOCKER], tunnel=False, direct_rc=100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "Could not resolve" in result["steps"][-1]["output"] + + def test_a_later_https_line_it_cannot_comment_out_fails(self): + host = _Host( + {"/etc/apt/apt.conf": 'Acquire::https { Proxy "http://c:3142"; };\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + host._effective = lambda scheme="http": "http://c:3142" + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"] + + def test_without_an_http_source_the_fetch_says_so(self): + host = _Host() + real = host._run_apt_command + + def no_sources(command): + return real(command).replace("http://deb.debian.org/debian/dists/trixie/\n", "") + + host._run_apt_command = no_sources + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "no http source" in _step(result, "fetch")["output"] + + def test_a_direct_source_is_not_proof(self): + host = _Host() + real = host._run_apt_command + host._run_apt_command = lambda command: real(command).replace( + "[parts]", 'Acquire::http::Proxy::deb.debian.org "DIRECT";\n[parts]' + ) + + result = host.configure_package_proxy("http://c:3142") + + assert "no http source" in _step(result, "fetch")["output"] + + @pytest.mark.parametrize("url", ['http://x"; Acquire::x "y', "ftp://c", "c:3142"]) + def test_a_url_that_would_break_the_file_is_refused(self, url): + host = _Host() + + result = host.configure_package_proxy(url) + + assert result["success"] is False + assert host.files == {} + + def test_an_empty_url_removes_the_file(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + + result = host.configure_package_proxy("") + + assert result["success"] is True + assert host.files == {} + + def test_only_an_apt_host_takes_one(self): + host = _Host(apt=False) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert host.files == {} + + +class TestWhatTheParserSkips: + def test_a_line_that_is_no_grep_output(self): + assert ( + parse_apt_proxy(_report(parts=["grep: /etc/apt/apt.conf.d/x: Permission denied"]))[ + "settings" + ] + == [] + ) + + def test_a_dump_line_of_another_key(self): + report = parse_apt_proxy(_report(dump=['Acquire::http::Timeout "10";'])) + assert report["proxy"] is None + + def test_a_fetch_report_without_an_exit_status_raises(self): + with pytest.raises(ValueError, match="exit status"): + parse_apt_fetch("APROXY_BEGIN\n[fetch]\nGet:1 x\nAPROXY_END\n") + + +class _Plain(AptProxyMixin): + """A driver that says nothing about its package manager: apt is assumed.""" + + +def test_without_its_own_answer_a_host_is_an_apt_host(): + assert _Plain()._is_apt_host() is True + + +class TestNotAnAptHost: + def test_no_check(self): + with pytest.raises(NotImplementedError): + _Host(apt=False).check_package_proxy("http://c:3142/x") + + def test_no_removal(self): + result = _Host(apt=False).configure_package_proxy("") + assert result["success"] is False + assert "not an apt host" in result["output"] + + +def test_a_comment_out_that_fails_stops_the_fix(): + host = _Host({f"{_D}90curtin-aptproxy": 'Acquire::http::Proxy "http://old:3128";\n'}) + host._run_apt_root_command = lambda command, done: {"ok": False, "output": "sed: denied"} + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1] == {"step": "conflicts", "ok": False, "output": "sed: denied"} + + +def test_the_comment_out_command(): + assert comment_out_command("/etc/apt/apt.conf", [2, 4]) == ( + "sed -i -e '2{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' " + "-e '4{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" + ) + + +def test_the_comment_out_command_for_https(): + assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == ( + "sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" + ) diff --git a/tests/test_arp_age.py b/tests/test_arp_age.py new file mode 100644 index 0000000..d913dd2 --- /dev/null +++ b/tests/test_arp_age.py @@ -0,0 +1,27 @@ +"""An ARP or neighbour entry whose age the device does not report says so. + +NAPALM's ``get_arp_table`` and ``get_ipv6_neighbors_table`` require an ``age`` +on every entry, and a driver that could not read one used to fill in ``0.0``. +To a caller that is the freshest sighting there is: netOrk took a neighbour the +device had not confirmed for hours as seen this second (NetOrk/netork#1327). +Unknown is ``-1.0``, as in NAPALM's own IOS driver; ``0.0`` means "just now". +""" + +from __future__ import annotations + +from napalm_device_types import ARP_AGE_UNKNOWN +from napalm_device_types.arp import ARP_AGE_UNKNOWN as FROM_MODULE + + +def test_an_unknown_age_is_minus_one(): + assert ARP_AGE_UNKNOWN == -1.0 + assert isinstance(ARP_AGE_UNKNOWN, float) + + +def test_the_package_exports_the_modules_constant(): + assert ARP_AGE_UNKNOWN is FROM_MODULE + + +def test_no_real_age_can_be_mistaken_for_unknown(): + # An age is seconds since the entry was last confirmed: never negative. + assert ARP_AGE_UNKNOWN < 0 diff --git a/tests/test_channel.py b/tests/test_channel.py index 83d23c2..954237a 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -8,8 +8,6 @@ paramiko channel is faked, so the tests pin behaviour, not paramiko. from __future__ import annotations -import socket - import pytest from napalm_device_types.channel import ( @@ -56,7 +54,7 @@ class FakeChannel: if self.out: return self.out.pop(0) if self.hang: - raise socket.timeout() + raise TimeoutError() return b"" def recv_stderr_ready(self): diff --git a/tests/test_clusters.py b/tests/test_clusters.py new file mode 100644 index 0000000..d678669 --- /dev/null +++ b/tests/test_clusters.py @@ -0,0 +1,168 @@ +"""A hypervisor says which cluster it belongs to, and what that cluster holds. + +A Proxmox node is one member of a cluster and can read the whole of it; a vCenter +manages several clusters and standalone hosts at once; a standalone host is a +cluster of one. One getter serves all three: ``get_clusters()`` lists every +cluster the connection can see, each with its nodes, its storages and its +networks, and which nodes see which. What a caller does with that (placement, +headroom, identity) is the caller's. + +A new VM can be placed on a named node (``create_vm_from_cloud_init(node=…)``), +so a manager that runs several hosts takes the caller's choice instead of making +its own. + +The declarations sit under ``TYPE_CHECKING`` (see test_role_contracts), so the +signatures are read from the source rather than from the class. +""" + +from __future__ import annotations + +import ast +import inspect +from typing import NotRequired, get_type_hints + +import napalm_device_types.hypervisor as hypervisor_module +from napalm_device_types import ( + BackupDict, + BackupJobDict, + ClusterDict, + ClusterNetworkDict, + ClusterNodeDict, + ClusterStorageDict, + HAResourceDict, + HypervisorDriver, +) + + +def _declared(name: str) -> ast.FunctionDef: + tree = ast.parse(inspect.getsource(hypervisor_module)) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == name: + return node + raise AssertionError(f"HypervisorDriver does not declare {name}()") + + +class TestClusterListing: + def test_is_declared(self): + fn = _declared("get_clusters") + assert ast.unparse(fn.returns) == "list[ClusterDict]" + + def test_absent_until_a_driver_implements_it(self): + """netOrk probes capabilities with hasattr; a hypervisor that cannot + describe its cluster must not seem to.""" + assert not hasattr(HypervisorDriver, "get_clusters") + + +class TestClusterShape: + def test_cluster(self): + assert get_type_hints(ClusterDict, include_extras=True) == { + "id": str | None, + "name": str | None, + "quorate": bool | None, + "nodes": list[ClusterNodeDict], + "storages": list[ClusterStorageDict], + "networks": list[ClusterNetworkDict], + "ha_resources": NotRequired[list[HAResourceDict] | None], + "backup_jobs": NotRequired[list[BackupJobDict] | None], + "backups": NotRequired[list[BackupDict] | None], + } + + def test_cluster_keys_added_later_stay_optional(self): + """A driver written before HA and backups were part of the report stays valid.""" + assert ClusterDict.__required_keys__ == frozenset( + {"id", "name", "quorate", "nodes", "storages", "networks"} + ) + + def test_node(self): + hints = get_type_hints(ClusterNodeDict, include_extras=True) + assert hints == { + "name": str, + "status": str, + "local": bool, + "maintenance": bool | None, + "cpu_threads": int | None, + "cpu_cores": int | None, + "cpu_model": str | None, + "cpu_flags": list[str] | None, + "cpu_usage": float | None, + "memory_total": int | None, + "memory_used": int | None, + "votes": NotRequired[int | None], + } + assert "votes" not in ClusterNodeDict.__required_keys__ + + def test_storage(self): + assert get_type_hints(ClusterStorageDict) == { + "name": str, + "type": str, + "shared": bool, + "content": list[str], + "nodes": list[str], + "total": int | None, + "used": int | None, + } + + def test_network(self): + hints = get_type_hints(ClusterNetworkDict, include_extras=True) + assert hints == { + "name": str, + "kind": str, + "vlan_aware": bool, + "fixed_vlan_tag": NotRequired[int | None], + "nodes": list[str], + } + assert ClusterNetworkDict.__required_keys__ == frozenset( + {"name", "kind", "vlan_aware", "nodes"} + ) + + +class TestHAAndBackupShapes: + def test_ha_resource(self): + hints = get_type_hints(HAResourceDict, include_extras=True) + assert hints == { + "vmid": int, + "kind": str, + "requested": str, + "status": str | None, + "node": str | None, + "group": NotRequired[str | None], + "rules": list[str], + } + assert HAResourceDict.__required_keys__ == frozenset( + {"vmid", "kind", "requested", "status", "node", "rules"} + ) + + def test_backup_job(self): + assert get_type_hints(BackupJobDict) == { + "id": str, + "enabled": bool, + "schedule": str | None, + "storage": str | None, + "mode": str | None, + "all": bool, + "vmids": list[int], + "exclude": list[int], + "pool": str | None, + "node": str | None, + } + + def test_backup(self): + assert get_type_hints(BackupDict) == { + "vmid": int, + "storage": str, + "time": int, + "size": int | None, + "count": int, + } + + +class TestCreateVmTakesANode: + def test_node_is_an_optional_keyword(self): + fn = _declared("create_vm_from_cloud_init") + kwonly = { + arg.arg: default + for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults, strict=True) + } + assert "node" in kwonly + default = kwonly["node"] + assert isinstance(default, ast.Constant) and default.value is None diff --git a/tests/test_container_engine.py b/tests/test_container_engine.py index 53199a0..24db7da 100644 --- a/tests/test_container_engine.py +++ b/tests/test_container_engine.py @@ -32,7 +32,9 @@ class FakeDriver(ContainerEngineMixin): def run_command(self, command, *, privileged=False, timeout=60, stdin=None): self.commands.append((command, privileged, timeout, stdin)) found = any(command == f"command -v {b}" for b in self._binaries()) - return CommandResult(stdout="/usr/bin/x\n" if found else "", stderr="", exit_code=0 if found else 1) + return CommandResult( + stdout="/usr/bin/x\n" if found else "", stderr="", exit_code=0 if found else 1 + ) def open_stream(self, command, *, privileged=False): self.streams.append((command, privileged)) @@ -86,7 +88,9 @@ def test_cli_arguments_are_quoted_and_prefixed_by_the_binary(): def test_cli_stdin_and_timeout_reach_the_channel(): driver = FakeDriver() - driver.open_container_engine("docker").run_cli(["login", "--password-stdin"], stdin=b"x", timeout=30) + driver.open_container_engine("docker").run_cli( + ["login", "--password-stdin"], stdin=b"x", timeout=30 + ) _command, _privileged, timeout, stdin = driver.commands[-1] assert (timeout, stdin) == (30, b"x") diff --git a/tests/test_dhcp_diff_apply.py b/tests/test_dhcp_diff_apply.py index 2d567ca..4aaef44 100644 --- a/tests/test_dhcp_diff_apply.py +++ b/tests/test_dhcp_diff_apply.py @@ -9,9 +9,10 @@ principle: generic vs. device-specific logic" for why this logic lives here and not in a vendor driver. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import DhcpServerMixin, FirewallDriver, ResidentialGatewayDriver from napalm_device_types.dhcp import normalize_mac from napalm_device_types.models import DhcpReservationDict @@ -33,21 +34,21 @@ def _reservation(**overrides: Any) -> DhcpReservationDict: class _FakeDhcpServer(DhcpServerMixin): """In-memory fake — no network, no Kea/UCI specifics.""" - def __init__(self, live: Optional[List[DhcpReservationDict]] = None) -> None: + def __init__(self, live: list[DhcpReservationDict] | None = None) -> None: self.live = live or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.commits = 0 - def get_dhcp_reservations(self) -> List[DhcpReservationDict]: + def get_dhcp_reservations(self) -> list[DhcpReservationDict]: return self.live def apply_dhcp_reservation( - self, reservation: DhcpReservationDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, reservation: DhcpReservationDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"reservation": reservation, "uuid": uuid}) return {"success": True} - def commit_dhcp_reservations(self) -> Dict[str, Any]: + def commit_dhcp_reservations(self) -> dict[str, Any]: self.commits += 1 return {"success": True} diff --git a/tests/test_dhcp_ha_config_sync.py b/tests/test_dhcp_ha_config_sync.py new file mode 100644 index 0000000..fbabb79 --- /dev/null +++ b/tests/test_dhcp_ha_config_sync.py @@ -0,0 +1,69 @@ +"""Two facts an HA pair of servers reports about itself. + +`get_dhcp_ha` is what a DHCP server's own HA configuration says: the peers it +pairs with and each peer's role, as the server words it. `get_config_sync` is +where a host pushes its configuration and for which of its services. Both are +declarations; what they mean for the pair -- hot standby, who to write to -- +is the caller's to work out. +""" + +from __future__ import annotations + +import inspect + +import napalm_device_types as package +from napalm_device_types import ConfigSyncMixin, DhcpServerMixin, FirewallDriver + + +class TestDeclaredOnly: + def test_get_dhcp_ha_is_absent_until_a_driver_implements_it(self): + assert not hasattr(DhcpServerMixin, "get_dhcp_ha") + + def test_get_config_sync_is_absent_until_a_driver_implements_it(self): + assert not hasattr(ConfigSyncMixin, "get_config_sync") + + def test_config_sync_mixin_owns_nothing_callable(self): + own = [ + name + for name, value in vars(ConfigSyncMixin).items() + if not name.startswith("__") and inspect.isfunction(value) + ] + assert own == [] + + def test_the_firewall_role_does_not_carry_config_sync(self): + """A driver mixes the capability in; the role does not promise it.""" + assert not issubclass(FirewallDriver, ConfigSyncMixin) + + def test_a_driver_that_implements_them_is_found_by_hasattr(self): + class Pair(ConfigSyncMixin, DhcpServerMixin): + def get_dhcp_ha(self): + return {"enabled": False, "peers": []} + + def get_config_sync(self): + return {"peer": None, "services": [], "unmapped": []} + + assert callable(getattr(Pair(), "get_dhcp_ha", None)) + assert callable(getattr(Pair(), "get_config_sync", None)) + + +class TestTypes: + def test_are_exported(self): + for name in ("ConfigSyncMixin", "ConfigSyncDict", "DhcpHaDict", "DhcpHaPeerDict"): + assert name in package.__all__ and hasattr(package, name), name + + def test_a_peer_needs_only_its_name_and_role(self): + """A future server may not have a URL to report; Kea does.""" + from napalm_device_types import DhcpHaPeerDict + + assert DhcpHaPeerDict.__required_keys__ == frozenset({"name", "role"}) + + def test_ha_needs_only_whether_it_is_on_and_the_peers(self): + from napalm_device_types import DhcpHaDict + + assert DhcpHaDict.__required_keys__ == frozenset({"enabled", "peers"}) + + def test_config_sync_reports_what_it_could_not_name(self): + """A sync item the driver has no service name for is still a fact.""" + from napalm_device_types import ConfigSyncDict + + assert ConfigSyncDict.__required_keys__ == frozenset({"peer", "services", "unmapped"}) diff --git a/tests/test_dhcp_subnet_diff_apply.py b/tests/test_dhcp_subnet_diff_apply.py index f478c42..39d3f1c 100644 --- a/tests/test_dhcp_subnet_diff_apply.py +++ b/tests/test_dhcp_subnet_diff_apply.py @@ -11,9 +11,10 @@ A subnet is a heavier object than a reservation: a wrong `pools` or below lean on the never-delete and preserve-unmanaged-options guarantees. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import DhcpServerMixin from napalm_device_types.models import DhcpSubnetDict @@ -35,21 +36,21 @@ def _subnet(**overrides: Any) -> DhcpSubnetDict: class _FakeDhcpServer(DhcpServerMixin): - def __init__(self, live: Optional[List[DhcpSubnetDict]] = None) -> None: + def __init__(self, live: list[DhcpSubnetDict] | None = None) -> None: self.live = live or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.commits = 0 - def get_dhcp_subnets(self) -> List[DhcpSubnetDict]: + def get_dhcp_subnets(self) -> list[DhcpSubnetDict]: return self.live def apply_dhcp_subnet( - self, subnet: DhcpSubnetDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, subnet: DhcpSubnetDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"subnet": subnet, "uuid": uuid}) return {"success": True} - def commit_dhcp_subnets(self) -> Dict[str, Any]: + def commit_dhcp_subnets(self) -> dict[str, Any]: self.commits += 1 return {"success": True} @@ -215,9 +216,7 @@ class _BareDriver(DhcpServerMixin): pass -@pytest.mark.parametrize( - "method", ["get_dhcp_subnets", "apply_dhcp_subnet", "commit_dhcp_subnets"] -) +@pytest.mark.parametrize("method", ["get_dhcp_subnets", "apply_dhcp_subnet", "commit_dhcp_subnets"]) def test_device_specific_methods_are_absent_until_implemented(method: str) -> None: """Declared under TYPE_CHECKING, so they do not exist until a driver adds them.""" assert not hasattr(_BareDriver, method) diff --git a/tests/test_fact_getters.py b/tests/test_fact_getters.py new file mode 100644 index 0000000..d9cc556 --- /dev/null +++ b/tests/test_fact_getters.py @@ -0,0 +1,63 @@ +"""Facts instead of warnings (docs/STRUCTURE.md, NetOrk/netork#1111). + +A driver reports what it reads; the application decides what is a warning. Three +facts that drivers so far only reported as warning codes get contracts of their +own: the package manager's proxy, licenses and subscriptions, and the state of a +hypervisor's hosts. All are declared only, so a capability check stays truthful. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types import ( + HypervisorDriver, + HypervisorHostDict, + LicenseDict, + LicenseMixin, + PackageProxyMixin, +) + + +def test_a_license_says_what_it_is_and_whether_it_holds(): + hints = get_type_hints(LicenseDict) + assert {"name", "status"} <= set(hints) + assert LicenseDict.__required_keys__ == frozenset({"name", "status"}) + assert set(hints) == {"name", "status", "edition", "expires", "detail", "withholds"} + + +def test_a_missing_license_may_say_what_the_device_withholds_without_it(): + # A Supermicro X10 with only SFT-OOB-LIC answers Redfish but keeps its drives and + # firmware inventory until DCMS is activated: partly licensed, not unlicensed. + hints = get_type_hints(LicenseDict) + assert hints["withholds"] == list[str] + assert "withholds" not in LicenseDict.__required_keys__ + + +def test_a_hypervisor_host_says_how_it_stands(): + hints = get_type_hints(HypervisorHostDict) + assert set(hints) == {"name", "connected", "maintenance", "config_issues", "managed_by"} + assert HypervisorHostDict.__required_keys__ == frozenset( + {"name", "connected", "maintenance", "config_issues"} + ) + + +def test_the_getters_are_declared_only(): + assert not hasattr(PackageProxyMixin, "get_package_proxy") + assert not hasattr(LicenseMixin, "get_licenses") + assert not hasattr(HypervisorDriver, "get_hypervisor_hosts") + + +def test_a_firmware_offer_says_what_runs_and_what_is_offered(): + """The vendor's newer firmware is a fact the driver can know (a download + catalogue, the device's own update check); whether it is a warning is the + application's call. Comparing vendor version strings stays with the driver, + so it says itself whether the offer is newer.""" + from napalm_device_types import FirmwareOfferDict, FirmwareOfferMixin + + hints = get_type_hints(FirmwareOfferDict) + assert set(hints) == {"current_version", "latest_version", "newer", "download_url"} + assert FirmwareOfferDict.__required_keys__ == frozenset( + {"current_version", "latest_version", "newer"} + ) + assert not hasattr(FirmwareOfferMixin, "get_firmware_offer") diff --git a/tests/test_fingerprint_base.py b/tests/test_fingerprint_base.py index 89738f9..3ad761d 100644 --- a/tests/test_fingerprint_base.py +++ b/tests/test_fingerprint_base.py @@ -3,21 +3,21 @@ from __future__ import annotations import pytest +from napalm.base import NetworkDriver + from napalm_device_types import ( AccessPointDriver, DeviceTypeDriver, FingerprintRule, FirewallDriver, - rules_max_weight, HypervisorDriver, OSDriver, PortSpec, ResidentialGatewayDriver, StorageDriver, SwitchDriver, + rules_max_weight, ) -from napalm.base import NetworkDriver - # ── Hierarchy ───────────────────────────────────────────────────────────────── @@ -49,6 +49,7 @@ def test_device_type_driver_inherits_network_driver(): # ── Default attributes ──────────────────────────────────────────────────────── + def test_defaults_on_device_type_driver(): assert DeviceTypeDriver.VENDOR == "" assert DeviceTypeDriver.DRIVER_NAME == "" @@ -72,6 +73,7 @@ def test_fingerprint_attributes_inherited(cls): # ── FingerprintRule ─────────────────────────────────────────────────────────── + def test_fingerprint_rule_defaults(): r = FingerprintRule("fritz!box") assert r.pattern == "fritz!box" @@ -102,6 +104,7 @@ def test_fingerprint_rule_group(): # ── rules_max_weight: the most a set of rules can add ───────────────────────── + def test_max_weight_sums_rules_that_can_match_together(): rules = [FingerprintRule("proxmox virtual environment", 9.0), FingerprintRule("pve", 2.0)] assert rules_max_weight(rules) == 11.0 @@ -146,6 +149,7 @@ def test_fingerprint_rule_is_immutable(): # ── PortSpec ────────────────────────────────────────────────────────────────── + def test_port_spec_defaults(): p = PortSpec("https", 8006) assert p.scheme == "https" @@ -169,6 +173,7 @@ def test_port_spec_is_immutable(): # ── Concrete driver subclass ────────────────────────────────────────────────── + def test_concrete_driver_overrides_fingerprint(): class MyDriver(FirewallDriver): VENDOR = "Acme" diff --git a/tests/test_firewall_diff_apply.py b/tests/test_firewall_diff_apply.py index 471972c..1001990 100644 --- a/tests/test_firewall_diff_apply.py +++ b/tests/test_firewall_diff_apply.py @@ -9,9 +9,10 @@ device-specific logic" for why this logic lives here and not in a vendor driver. """ -from typing import Any, Dict, List, Optional +from typing import Any import pytest + from napalm_device_types import FirewallDriver from napalm_device_types.models import FirewallRuleDict @@ -39,21 +40,21 @@ def _rule(**overrides: Any) -> FirewallRuleDict: class _FakeFirewall(FirewallDriver): """In-memory fake — no network, no OPNsense/vendor specifics.""" - def __init__(self, live_rules: Optional[List[FirewallRuleDict]] = None) -> None: + def __init__(self, live_rules: list[FirewallRuleDict] | None = None) -> None: self.live_rules = live_rules or [] - self.applied: List[Dict[str, Any]] = [] + self.applied: list[dict[str, Any]] = [] self.committed = False - def get_firewall_rules(self) -> List[FirewallRuleDict]: + def get_firewall_rules(self) -> list[FirewallRuleDict]: return self.live_rules def apply_firewall_rule( - self, rule: FirewallRuleDict, *, uuid: Optional[str] = None - ) -> Dict[str, Any]: + self, rule: FirewallRuleDict, *, uuid: str | None = None + ) -> dict[str, Any]: self.applied.append({"rule": rule, "uuid": uuid}) return {"success": True} - def commit_firewall_rules(self) -> Dict[str, Any]: + def commit_firewall_rules(self) -> dict[str, Any]: self.committed = True return {"success": True} diff --git a/tests/test_fix_templates.py b/tests/test_fix_templates.py new file mode 100644 index 0000000..6c1adfc --- /dev/null +++ b/tests/test_fix_templates.py @@ -0,0 +1,268 @@ +"""Fix routines are templates (docs/STRUCTURE.md, NetOrk/netork#1105). + +The template fixes the order of the steps, the driver knows how each step is done +on its device, the caller supplies every value. None of them is mixed into a role +base, so ``callable(getattr(driver, name, None))`` stays a truthful answer. +""" + +from __future__ import annotations + +import pytest + +from napalm_device_types import ( + AccessPointDriver, + EnsurePackageMixin, + FirewallDriver, + HypervisorDriver, + LldpAgentMixin, + ManagementControllerDriver, + MediaDriver, + OSDriver, + PackageProxyMixin, + PhoneDriver, + ResidentialGatewayDriver, + SnmpAgentMixin, + StorageDriver, + SwitchDriver, + run_fix_steps, +) + +ROLE_BASES = [ + AccessPointDriver, FirewallDriver, HypervisorDriver, ManagementControllerDriver, MediaDriver, + OSDriver, PhoneDriver, ResidentialGatewayDriver, StorageDriver, SwitchDriver, +] # fmt: skip + + +def _ok(output: str = ""): + return lambda: {"ok": True, "output": output} + + +def _fail(output: str): + return lambda: {"ok": False, "output": output} + + +class TestRunFixSteps: + def test_every_step_runs_in_order_and_is_reported(self): + result = run_fix_steps([("install", _ok("installed")), ("start", _ok("started"))]) + + assert result["success"] is True + assert result["steps"] == [ + {"step": "install", "ok": True, "output": "installed"}, + {"step": "start", "ok": True, "output": "started"}, + ] + assert result["output"] == "[install] installed\n[start] started" + + def test_the_first_failure_stops_the_rest(self): + ran = [] + + def later(): + ran.append("later") + return {"ok": True, "output": ""} + + result = run_fix_steps([("install", _fail("no package")), ("start", later)]) + + assert result["success"] is False + assert [s["step"] for s in result["steps"]] == ["install"] + assert ran == [] + + def test_a_step_that_raises_is_a_failed_step(self): + def broken(): + raise RuntimeError("connection lost") + + result = run_fix_steps([("install", broken)]) + + assert result["success"] is False + assert result["steps"] == [{"step": "install", "ok": False, "output": "connection lost"}] + + +class _Agent(SnmpAgentMixin): + def __init__(self, fail_at: str = ""): + self.calls: list = [] + self.fail_at = fail_at + + def _step(self, name, *args): + self.calls.append((name, *args)) + return {"ok": name != self.fail_at, "output": name} + + def _install_snmp_agent(self): + return self._step("install") + + def _write_snmp_agent_config(self, settings): + return self._step("configure", settings) + + def _allow_snmp_sources(self, settings): + return self._step("firewall", settings) + + def _start_snmp_agent(self): + return self._step("start") + + +class TestSnmpAgent: + SETTINGS = { + "community": "c0mm", + "allowed_sources": ["10.0.0.5"], + "contact": "ops", + "location": "rack 1", + } + + def test_install_configure_open_and_start_with_the_caller_s_values(self): + agent = _Agent() + + result = agent.configure_snmp_agent(self.SETTINGS) + + assert result["success"] is True + assert agent.calls == [ + ("install",), + ("configure", self.SETTINGS), + ("firewall", self.SETTINGS), + ("start",), + ] + + def test_a_failed_install_configures_nothing(self): + agent = _Agent(fail_at="install") + + assert agent.configure_snmp_agent(self.SETTINGS)["success"] is False + assert agent.calls == [("install",)] + + +class _Packages(EnsurePackageMixin): + def __init__(self, installed): + self.installed = installed + self.asked: list = [] + + def get_packages(self): + return [{"name": n, "installed": True} for n in self.installed] + [ + {"name": "lldpd", "installed": False} + ] + + def install_package(self, name, version=""): + self.asked.append(name) + + +class TestEnsurePackage: + def test_installs_what_is_missing(self): + driver = _Packages(installed=["curl"]) + + result = driver.ensure_package("lldpd") + + assert result["success"] is True + assert driver.asked == ["lldpd"] + + def test_leaves_alone_what_is_there(self): + driver = _Packages(installed=["lldpd"]) + + result = driver.ensure_package("lldpd") + + assert result["success"] is True + assert driver.asked == [] + assert "already installed" in result["output"] + + def test_an_install_that_reports_failure_fails(self): + driver = _Packages(installed=[]) + driver.install_package = lambda name, version="": { + "success": False, + "output": "E: no candidate", + } + + result = driver.ensure_package("lldpd") + + assert result["success"] is False + assert "no candidate" in result["output"] + + +class _Lldp(LldpAgentMixin): + """The generic steps over ensure_package and manage_service; the driver's configure.""" + + def __init__(self, refuse: str = ""): + self.calls: list = [] + self.refuse = refuse + + def ensure_package(self, name): + self.calls.append(("ensure_package", name)) + ok = self.refuse != "install" + return {"success": ok, "steps": [], "output": "installed" if ok else "E: no candidate"} + + def _write_lldp_agent_config(self, interfaces): + self.calls.append(("configure", interfaces)) + return {"ok": self.refuse != "configure", "output": "interfaces set"} + + def manage_service(self, name, action): + self.calls.append((action, name)) + ok = self.refuse != action + return {"success": ok, "output": "" if ok else f"cannot {action}"} + + +class TestLldpAgent: + def test_install_configure_enable_and_restart_on_the_caller_s_interfaces(self): + agent = _Lldp() + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is True + assert agent.calls == [ + ("ensure_package", "lldpd"), + ("configure", ["br-lan"]), + ("enable", "lldpd"), + ("restart", "lldpd"), + ] + assert [s["step"] for s in result["steps"]] == ["install", "configure", "start"] + + def test_a_failed_install_configures_nothing(self): + agent = _Lldp(refuse="install") + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert agent.calls == [("ensure_package", "lldpd")] + assert "no candidate" in result["output"] + + def test_an_agent_that_will_not_start_at_boot_is_not_restarted(self): + agent = _Lldp(refuse="enable") + + result = agent.configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert ("restart", "lldpd") not in agent.calls + assert "cannot enable" in result["output"] + + def test_a_failed_restart_fails_the_fix(self): + result = _Lldp(refuse="restart").configure_lldp_agent(["br-lan"]) + + assert result["success"] is False + assert result["steps"][-1]["step"] == "start" + + def test_the_package_and_service_names_are_the_driver_s(self): + agent = _Lldp() + agent.LLDP_AGENT_PACKAGE = "lldpd-full" + agent.LLDP_AGENT_SERVICE = "lldpd.service" + + agent.configure_lldp_agent(["eth0"]) + + assert ("ensure_package", "lldpd-full") in agent.calls + assert ("restart", "lldpd.service") in agent.calls + + def test_no_interface_is_refused_before_anything_changes(self): + agent = _Lldp() + + with pytest.raises(ValueError): + agent.configure_lldp_agent([]) + assert agent.calls == [] + + +@pytest.mark.parametrize("base", ROLE_BASES, ids=lambda b: b.__name__) +@pytest.mark.parametrize( + "method", + [ + "configure_snmp_agent", + "configure_lldp_agent", + "ensure_package", + "configure_package_proxy", + "grant_container_engine_access", + ], +) +def test_no_role_base_offers_a_fix(base, method): + assert getattr(base, method, None) is None + + +def test_the_package_proxy_is_declared_only(): + assert not hasattr(PackageProxyMixin, "configure_package_proxy") diff --git a/tests/test_host_firewall.py b/tests/test_host_firewall.py new file mode 100644 index 0000000..2bcd805 --- /dev/null +++ b/tests/test_host_firewall.py @@ -0,0 +1,501 @@ +"""get_host_firewall: a Linux host's own netfilter rules, read once, in one shape. + +Whether a service on a host is reachable from the internet is not only decided +by the address it listens on: the host's own firewall may drop the packets +before they reach it. Reading that is the same on every Linux host, so the +command and the parse live here, and a driver only carries the command across +(NetOrk/netork#1344). + +``nft -j list ruleset`` is the primary reading. Rules that iptables-nft wrote +show their extensions there only as an opaque ``xt`` (``-m conntrack``, +``-m set``), so the tables iptables manages are taken from ``iptables-save``, +which spells them out. A legacy iptables is a ruleset of its own beside nft, +and both apply. +""" + +from __future__ import annotations + +import json +from shlex import quote + +import pytest + +from napalm_device_types.host_firewall import ( + HOST_FIREWALL_COMMAND, + HostFirewallMixin, + HostFirewallUnavailable, + parse_host_firewall, +) + +IPTABLES_NFT = """# Generated by iptables-save v1.8.10 (nf_tables) on Wed Oct 21 14:52:11 2026 +*filter +:INPUT ACCEPT [2783578:295175610] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +:DOCKER-USER - [0:0] +:WGRUNNER - [0:0] +-A INPUT -s 10.255.255.4/32 -j WGRUNNER +-A INPUT -i ens6 -p udp -m udp --dport 51820 -j ACCEPT +-A INPUT -m set --match-set crowdsec-blacklists src -j DROP +-A INPUT -i ens6 -m state --state RELATED,ESTABLISHED -j ACCEPT +-A INPUT -i ens6 -p tcp -m multiport --dports 80,443,8000:8010 -j ACCEPT +-A INPUT ! -i lo -p tcp -m conntrack --ctstate NEW --ctorigdstport 2222 -j LOG --log-prefix "ssh " +-A INPUT -i ens6 -j DROP +-A FORWARD -j DOCKER-USER +-A DOCKER-USER -i eth+ ! -s 192.168.0.0/16 -g WGRUNNER +-A WGRUNNER -d 10.7.224.12/32 -p tcp -m tcp --dport 10051 -j ACCEPT +-A WGRUNNER -j REJECT --reject-with icmp-port-unreachable +-A WGRUNNER -j RETURN +COMMIT +# Completed on Wed Oct 21 14:52:11 2026 +*nat +:PREROUTING ACCEPT [0:0] +:POSTROUTING ACCEPT [0:0] +:DOCKER - [0:0] +-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER +-A DOCKER ! -i br-1 -p udp -m udp --dport 53 -j DNAT --to-destination 172.18.0.2:53 +COMMIT +""" + +IP6TABLES_NFT = """# Generated by ip6tables-save v1.8.10 (nf_tables) on Wed Oct 21 14:52:11 2026 +*filter +:INPUT DROP [0:0] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -i lo -j ACCEPT +COMMIT +""" + + +def _nft(*objects: dict) -> str: + meta = {"metainfo": {"version": "1.0.9", "json_schema_version": 1}} + return json.dumps({"nftables": [meta, *objects]}) + + +#: The iptables-nft filter table as nft shows it -- with its extensions opaque -- +#: and a native ``inet filter`` table of the host's own nftables.service. +NFT = _nft( + {"table": {"family": "ip", "name": "filter", "handle": 1}}, + { + "chain": { + "family": "ip", + "table": "filter", + "name": "INPUT", + "type": "filter", + "hook": "input", + "prio": 0, + "policy": "accept", + } + }, + { + "rule": { + "family": "ip", + "table": "filter", + "chain": "INPUT", + "expr": [ + {"xt": {"type": "match", "name": "set"}}, + {"counter": {"packets": 0, "bytes": 0}}, + {"drop": None}, + ], + } + }, + {"table": {"family": "inet", "name": "host", "handle": 9}}, + { + "set": { + "family": "inet", + "table": "host", + "name": "admins", + "type": "ipv4_addr", + "flags": ["interval"], + "elem": [{"prefix": {"addr": "198.51.100.0", "len": 24}}], + } + }, + { + "chain": { + "family": "inet", + "table": "host", + "name": "input", + "type": "filter", + "hook": "input", + "prio": 10, + "policy": "drop", + } + }, + {"chain": {"family": "inet", "table": "host", "name": "services"}}, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "in", + "left": {"ct": {"key": "state"}}, + "right": ["established", "related"], + } + }, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "==", "left": {"meta": {"key": "iif"}}, "right": "lo"}}, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "ip", "field": "saddr"}}, + "right": "@admins", + } + }, + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "tcp", "field": "dport"}}, + "right": {"set": [22, {"range": [8000, 8010]}]}, + } + }, + {"accept": None}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "!=", "left": {"meta": {"key": "iifname"}}, "right": "wg*"}}, + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "ip6", "field": "daddr"}}, + "right": {"prefix": {"addr": "2001:db8::", "len": 32}}, + } + }, + {"jump": {"target": "services"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + {"match": {"op": "==", "left": {"meta": {"key": "l4proto"}}, "right": "udp"}}, + {"limit": {"rate": 10, "per": "second"}}, + {"log": {"prefix": "udp "}}, + {"reject": {"type": "icmpx", "expr": "port-unreachable"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "input", + "expr": [ + { + "match": { + "op": "==", + "left": {"fib": {"result": "type", "flags": ["daddr"]}}, + "right": "local", + } + }, + {"goto": {"target": "services"}}, + ], + } + }, + { + "rule": { + "family": "inet", + "table": "host", + "chain": "services", + "expr": [ + { + "match": { + "op": "==", + "left": {"payload": {"protocol": "udp", "field": "dport"}}, + "right": 53, + } + }, + {"queue": {"num": 1}}, + ], + } + }, +) + + +def _report( + *, + nft: str | None = NFT, + nft_rc: int = 0, + ipt: str | None = IPTABLES_NFT, + ip6t: str | None = IP6TABLES_NFT, +) -> str: + parts = ["$ sh -c ...", "FW_BEGIN"] + if nft is not None: + parts += ["[nft]", nft, f"__FW_RC={nft_rc}"] + if ipt is not None: + parts += ["[iptables-save]", ipt, "__FW_RC=0"] + if ip6t is not None: + parts += ["[ip6tables-save]", ip6t, "__FW_RC=0"] + parts.append("FW_END") + return "\n".join(parts) + "\n" + + +def _chain(reading, family, table, name): + (chain,) = [ + c + for c in reading["chains"] + if (c["family"], c["table"], c["name"]) == (family, table, name) + ] + return chain + + +def _fields(rule): + return [(m["field"], m["op"], m["values"]) for m in rule["matches"]] + + +class TestTheSources: + def test_iptables_tables_come_from_iptables_save_the_rest_from_nft(self): + reading = parse_host_firewall(_report()) + + assert reading["sources"] == ["nftables", "iptables-nft"] + tables = {(c["family"], c["table"]) for c in reading["chains"]} + assert tables == {("ip", "filter"), ("ip", "nat"), ("ip6", "filter"), ("inet", "host")} + # The filter table's INPUT is iptables-save's: its set match is spelled + # out there, and nft's opaque copy is not added a second time. + assert len(_chain(reading, "ip", "filter", "INPUT")["rules"]) == 7 + + def test_a_legacy_iptables_is_a_ruleset_of_its_own(self): + legacy = IPTABLES_NFT.replace("(nf_tables)", "(legacy)") + + reading = parse_host_firewall(_report(ipt=legacy, ip6t=None)) + + assert reading["sources"] == ["nftables", "iptables-legacy"] + inputs = [c for c in reading["chains"] if c["name"] == "INPUT" and c["family"] == "ip"] + assert len(inputs) == 2 # nft's own ip filter INPUT and legacy's both apply + + def test_nft_alone(self): + reading = parse_host_firewall(_report(ipt=None, ip6t=None)) + + assert reading["sources"] == ["nftables"] + assert _chain(reading, "ip", "filter", "INPUT")["rules"][0]["matches"] == [ + {"field": "unknown", "op": "==", "values": ["xt match set"]} + ] + + def test_iptables_alone_when_nft_fails(self): + reading = parse_host_firewall(_report(nft="Operation not permitted", nft_rc=1)) + + assert reading["sources"] == ["iptables-nft"] + + def test_no_tool_that_could_be_read(self): + with pytest.raises(HostFirewallUnavailable): + parse_host_firewall(_report(nft=None, ipt=None, ip6t=None)) + + def test_a_failed_reading_is_no_reading(self): + with pytest.raises(HostFirewallUnavailable): + parse_host_firewall(_report(nft="permission denied", nft_rc=1, ipt=None, ip6t=None)) + + def test_a_report_cut_short_raises(self): + with pytest.raises(ValueError): + parse_host_firewall(_report().replace("FW_END", "")) + + def test_an_empty_ruleset_is_a_host_without_rules(self): + reading = parse_host_firewall(_report(nft=_nft(), ipt=None, ip6t=None)) + + assert reading == {"sources": ["nftables"], "chains": []} + + +class TestIptablesSave: + def setup_method(self): + self.reading = parse_host_firewall(_report(nft=None)) + + def test_a_built_in_chain_has_its_hook_and_policy(self): + chain = _chain(self.reading, "ip", "filter", "INPUT") + assert (chain["type"], chain["hook"], chain["priority"], chain["policy"]) == ( + "filter", + "input", + 0, + "accept", + ) + assert _chain(self.reading, "ip6", "filter", "INPUT")["policy"] == "drop" + nat = _chain(self.reading, "ip", "nat", "PREROUTING") + assert (nat["type"], nat["hook"], nat["priority"]) == ("nat", "prerouting", -100) + + def test_a_user_chain_has_none(self): + chain = _chain(self.reading, "ip", "filter", "WGRUNNER") + assert (chain["type"], chain["hook"], chain["priority"], chain["policy"]) == ( + None, + None, + None, + None, + ) + + def test_matches_and_verdicts(self): + rules = _chain(self.reading, "ip", "filter", "INPUT")["rules"] + assert _fields(rules[0]) == [("saddr", "==", ["10.255.255.4/32"])] + assert (rules[0]["verdict"], rules[0]["target"]) == ("jump", "WGRUNNER") + assert _fields(rules[1]) == [ + ("iifname", "==", ["ens6"]), + ("l4proto", "==", ["udp"]), + ("dport", "==", ["51820"]), + ] + assert rules[1]["verdict"] == "accept" + assert _fields(rules[2]) == [ + ("unknown", "==", ["-m set --match-set crowdsec-blacklists src"]), + ] + assert _fields(rules[3]) == [ + ("iifname", "==", ["ens6"]), + ("ct_state", "==", ["related", "established"]), + ] + assert _fields(rules[4]) == [ + ("iifname", "==", ["ens6"]), + ("l4proto", "==", ["tcp"]), + ("dport", "==", ["80", "443", "8000-8010"]), + ] + assert _fields(rules[5]) == [ + ("iifname", "!=", ["lo"]), + ("l4proto", "==", ["tcp"]), + ("ct_state", "==", ["new"]), + ("ct_orig_dport", "==", ["2222"]), + ] + assert rules[5]["verdict"] == "continue" # LOG decides nothing + assert rules[6]["verdict"] == "drop" + + def test_negation_wildcard_and_goto(self): + (rule,) = _chain(self.reading, "ip", "filter", "DOCKER-USER")["rules"] + assert _fields(rule) == [("iifname", "==", ["eth*"]), ("saddr", "!=", ["192.168.0.0/16"])] + assert (rule["verdict"], rule["target"]) == ("goto", "WGRUNNER") + + def test_reject_return_and_nat_targets(self): + rules = _chain(self.reading, "ip", "filter", "WGRUNNER")["rules"] + assert [r["verdict"] for r in rules] == ["accept", "reject", "return"] + (dnat,) = _chain(self.reading, "ip", "nat", "DOCKER")["rules"] + assert (dnat["verdict"], dnat["target"]) == ("other", "DNAT") + assert _chain(self.reading, "ip", "nat", "PREROUTING")["rules"][0]["matches"] == [ + {"field": "unknown", "op": "==", "values": ["-m addrtype --dst-type LOCAL"]} + ] + + def test_the_line_is_kept_as_the_rules_text(self): + rule = _chain(self.reading, "ip", "filter", "INPUT")["rules"][6] + assert rule["text"] == "-A INPUT -i ens6 -j DROP" + + +class TestNftJson: + def setup_method(self): + self.chains = { + c["name"]: c + for c in parse_host_firewall(_report(ipt=None, ip6t=None))["chains"] + if c["table"] == "host" + } + + def test_a_base_chain_and_a_regular_one(self): + base, regular = self.chains["input"], self.chains["services"] + assert (base["family"], base["type"], base["hook"], base["priority"], base["policy"]) == ( + "inet", + "filter", + "input", + 10, + "drop", + ) + assert (regular["hook"], regular["policy"]) == (None, None) + + def test_matches(self): + rules = self.chains["input"]["rules"] + assert _fields(rules[0]) == [("ct_state", "==", ["established", "related"])] + assert _fields(rules[1]) == [("iifname", "==", ["lo"])] + # A named set is resolved from its elements; tcp dport implies tcp. + assert _fields(rules[2]) == [ + ("saddr", "==", ["198.51.100.0/24"]), + ("l4proto", "==", ["tcp"]), + ("dport", "==", ["22", "8000-8010"]), + ] + assert _fields(rules[3]) == [("iifname", "!=", ["wg*"]), ("daddr", "==", ["2001:db8::/32"])] + assert (rules[3]["verdict"], rules[3]["target"]) == ("jump", "services") + + def test_a_statement_that_takes_only_some_packets_is_unknown(self): + rule = self.chains["input"]["rules"][4] + assert _fields(rule) == [("l4proto", "==", ["udp"]), ("unknown", "==", ["limit"])] + assert rule["verdict"] == "reject" + + def test_an_expression_it_does_not_model_is_unknown(self): + rule = self.chains["input"]["rules"][5] + assert rule["matches"][0]["field"] == "unknown" + assert (rule["verdict"], rule["target"]) == ("goto", "services") + + def test_a_verdict_it_does_not_model_is_other(self): + (rule,) = self.chains["services"]["rules"] + assert rule["verdict"] == "other" + + def test_each_rule_reads_as_a_line(self): + assert self.chains["input"]["rules"][2]["text"] == ( + "ip saddr 198.51.100.0/24 meta l4proto tcp tcp dport {22, 8000-8010} accept" + ) + + def test_an_unresolvable_set_is_unknown(self): + nft = NFT.replace('"@admins"', '"@missing"') + rules = [ + c + for c in parse_host_firewall(_report(nft=nft, ipt=None, ip6t=None))["chains"] + if c["name"] == "input" + ][0]["rules"] + assert rules[2]["matches"][0] == { + "field": "unknown", + "op": "==", + "values": ["ip saddr @missing"], + } + + +class _Host(HostFirewallMixin): + def __init__(self, output: str | None = None, fail: Exception | None = None): + self.calls: list[tuple[str, bool]] = [] + self._output, self._fail = output, fail + + def _run_host_firewall_command(self, command: str, *, privileged: bool) -> str: + self.calls.append((command, privileged)) + if self._fail: + raise self._fail + return self._output or "" + + +class TestTheMixin: + def test_one_privileged_command(self): + host = _Host(_report()) + + reading = host.get_host_firewall() + + assert host.calls == [(f"sh -c {quote(HOST_FIREWALL_COMMAND)}", True)] + assert reading["sources"] == ["nftables", "iptables-nft"] + + def test_without_root_there_is_nothing_to_read(self): + # nft and iptables-save both need CAP_NET_ADMIN; a reading without root + # would be an empty ruleset that is not the host's. + host = _Host("FW_BEGIN\n[nft]\nnetlink: Operation not permitted\n__FW_RC=1\nFW_END\n") + + with pytest.raises(HostFirewallUnavailable): + host.get_host_firewall() + assert len(host.calls) == 1 + + def test_the_command_is_framed_and_read_only(self): + assert "nft -j list ruleset" in HOST_FIREWALL_COMMAND + assert "iptables-save" in HOST_FIREWALL_COMMAND + for word in (" -F", " -X", "flush", "delete", "-A ", "-I ", "restore"): + assert word not in HOST_FIREWALL_COMMAND diff --git a/tests/test_host_shutdown.py b/tests/test_host_shutdown.py new file mode 100644 index 0000000..0a9182a --- /dev/null +++ b/tests/test_host_shutdown.py @@ -0,0 +1,27 @@ +"""shutdown_host: the contract for shutting the device itself down cleanly. + +netOrk's graph workflows switch a hypervisor off when it has nothing to do +(NetOrk/netork#1636): through its own API, never by pulling the power. Like +``reboot_host`` it is declared, not implemented, so ``hasattr`` tells netOrk +whether a driver can. +""" + +from __future__ import annotations + +import inspect + +from napalm_device_types import DeviceTypeDriver, HostShutdownMixin + + +def test_every_device_type_driver_carries_the_declaration(): + assert issubclass(DeviceTypeDriver, HostShutdownMixin) + + +def test_declared_not_implemented(): + assert not hasattr(DeviceTypeDriver, "shutdown_host") + + +def test_declaration_documents_the_contract(): + source = inspect.getsource(HostShutdownMixin) + assert "def shutdown_host(self) -> None" in source + assert "RuntimeError" in source diff --git a/tests/test_host_status.py b/tests/test_host_status.py index 1b3341c..2465e46 100644 --- a/tests/test_host_status.py +++ b/tests/test_host_status.py @@ -452,7 +452,10 @@ class TestBusyCheck: UBUNTU_IDLE = [ ("systemd", "/sbin/init"), - ("unattended-upgr", "/usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal"), + ( + "unattended-upgr", + "/usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal", + ), ("sshd", "sshd: netork [priv]"), ] diff --git a/tests/test_lag_interfaces.py b/tests/test_lag_interfaces.py index 852655c..f144936 100644 --- a/tests/test_lag_interfaces.py +++ b/tests/test_lag_interfaces.py @@ -3,7 +3,9 @@ from napalm_device_types import add_lag_interfaces -def _port(is_up: bool = True, is_enabled: bool = True, speed: float = 1000.0, trunk_group: str = "") -> dict: +def _port( + is_up: bool = True, is_enabled: bool = True, speed: float = 1000.0, trunk_group: str = "" +) -> dict: port = { "is_up": is_up, "is_enabled": is_enabled, diff --git a/tests/test_listening_sockets.py b/tests/test_listening_sockets.py index b67954f..e808637 100644 --- a/tests/test_listening_sockets.py +++ b/tests/test_listening_sockets.py @@ -89,7 +89,7 @@ class TestParsing: ], ) def test_every_address_form(self, local: str, address: str, interface): - line = f"tcp LISTEN 0 128 {local} *:* users:((\"x\",pid=1,fd=3))\n" + line = f'tcp LISTEN 0 128 {local} *:* users:(("x",pid=1,fd=3))\n' [socket] = parse_listening_sockets(_wire(line, "1 0::/system.slice/x.service\n")) @@ -188,7 +188,7 @@ class TestFailures: def test_a_report_cut_short_raises(self): with pytest.raises(ValueError): - parse_listening_sockets(_wire()[:-len("SOCK_END\n")]) + parse_listening_sockets(_wire()[: -len("SOCK_END\n")]) def test_a_host_without_ss_or_netstat_is_unavailable(self): with pytest.raises(ListeningSocketsUnavailable): diff --git a/tests/test_local_accounts.py b/tests/test_local_accounts.py new file mode 100644 index 0000000..30f1363 --- /dev/null +++ b/tests/test_local_accounts.py @@ -0,0 +1,692 @@ +"""Personal accounts on a host: reading them, adding one, locking one again. + +The host is a fake that keeps accounts, groups, sudo rules and key files in +memory and answers the fixed commands the mixin and a driver send. A driver's +own commands (create, open, lock, leave a group) are stand-ins here; the real +ones are tested in the drivers. +""" + +from __future__ import annotations + +import shlex +from dataclasses import dataclass, field + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.local_accounts import ( + LocalAccountsMixin, + authorized_keys_paths, + sudoers_rule, + sudoers_rule_path, +) +from napalm_device_types.models import FixStepOutcomeDict + +ALICE_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAlice alice@laptop" +OWN_KEY = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQOwn own@desktop" +HASH = "$6$salt$" + "x" * 86 +#: Shell syntax a command for root must not hold: each is one fixed command. +SHELL_SYNTAX = (";", "&&", "||", "|", ">", "<", "$(", "`") + + +@dataclass +class _Account: + uid: int + home: str + shell: str = "/bin/sh" + groups: list[str] = field(default_factory=list) + locked: bool = False + password: str | None = None + + +class _Host: + """What the fake host holds, and the commands it ran.""" + + def __init__(self) -> None: + self.login = "netadmin" + self.accounts: dict[str, _Account] = { + "netadmin": _Account(1000, "/home/netadmin", groups=["netadmin", "wheel"]), + "daemon": _Account(1, "/usr/sbin"), + } + #: Path -> content, for the sudoers drop-ins and the key files. + self.files: dict[str, str] = {} + #: Users a rule outside the drop-in directory lets run everything. + self.other_rules: set[str] = set() + #: Whether sudo reads the drop-in directory at all. + self.includes_dropins = True + #: Commands that fail, by their first words ("as " for one run as + #: the person); and commands that succeed without doing anything. + self.failing: set[str] = set() + self.ignored: set[str] = set() + self.ran: list[tuple[str, bool]] = [] + self.staged = 0 + + def is_admin(self, name: str) -> bool: + rule = self.files.get(sudoers_rule_path(_Driver.SUDOERS_DIR, name)) + return name in self.other_rules or (self.includes_dropins and rule is not None) + + def run(self, command: str, privileged: bool, stdin: bytes | None) -> CommandResult: + self.ran.append((command, privileged)) + if stdin is not None: + assert not privileged, "a privileged command takes no stdin" + return self._stage(command, stdin) + words = shlex.split(command) + if " ".join(words[:2]) in self.failing or words[0] in self.failing: + return CommandResult("", f"{words[0]}: failed", 1) + if words[:3] == ["sudo", "-n", "-u"]: + assert privileged, "only root changes to another user" + return self._as_user(words[3], words[4:]) + handler = getattr(self, "_" + words[0].replace("-", "_"), None) + assert handler is not None, f"unexpected command: {command}" + return handler(words[1:], privileged) + + # -- staging, as the login user --------------------------------------------- + + def _stage(self, command: str, stdin: bytes) -> CommandResult: + assert "mktemp" in command + if "mktemp" in self.failing: + return CommandResult("", "mktemp: No space left on device", 1) + self.staged += 1 + path = f"/tmp/tmp.{self.staged}" + self.files[path] = stdin.decode() + return CommandResult(path, "", 0) + + def _rm(self, args: list[str], privileged: bool) -> CommandResult: + for path in args[args.index("--") + 1 :]: + if path.startswith("/tmp/"): + assert not privileged, "the staged file is the login user's" + self.files.pop(path, None) + return CommandResult("", "", 0) + + # -- the account database ------------------------------------------------------- + + def _getent(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "passwd" and not privileged + a = self.accounts.get(args[1]) + if a is None: + return CommandResult("", "", 2) + return CommandResult(f"{args[1]}:*:{a.uid}:{a.uid}::{a.home}:{a.shell}\n", "", 0) + + def _id(self, args: list[str], privileged: bool) -> CommandResult: + if args == ["-un"]: + return CommandResult(f"{self.login}\n", "", 0) + assert args[0] == "-Gn" and not privileged + a = self.accounts.get(args[1]) + if a is None: + return CommandResult("", f"id: {args[1]}: no such user", 1) + return CommandResult(" ".join(a.groups) + "\n", "", 0) + + def _sudo(self, args: list[str], privileged: bool) -> CommandResult: + assert args[:2] == ["-l", "-U"] and privileged + name = args[2] + if self.is_admin(name): + out = f"User {name} may run the following commands on host:\n (ALL) ALL\n" + else: + out = f"User {name} is not allowed to run sudo on host.\n" + return CommandResult(out, "", 0) + + def _visudo(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "-cf" and privileged + content = self.files[args[1]] + ok = content.endswith(" ALL=(ALL) ALL\n") and "\n" not in content[:-1] + return CommandResult("parsed OK" if ok else "syntax error", "", 0 if ok else 1) + + def _install(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + source, target = args[args.index("--") + 1 :] + assert target.startswith(_Driver.SUDOERS_DIR + "/"), "root installs only sudo rules" + assert args[:6] == ["-m", "440", "-o", "root", "-g", _Driver.ROOT_GROUP] + self.files[target] = self.files[source] + return CommandResult("", "", 0) + + def _test(self, args: list[str], privileged: bool) -> CommandResult: + assert args[0] == "-e" + return CommandResult("", "", 0 if args[1] in self.files else 1) + + # -- the person's own files, as that person ------------------------------------ + + def _as_user(self, name: str, words: list[str]) -> CommandResult: + a = self.accounts[name] + if a.locked: + return CommandResult("", "sudo: account expired", 1) + rest = words[words.index("--") + 1 :] if "--" in words else words[1:] + if f"as {words[0]}" in self.failing: + return CommandResult("", f"{words[0]}: Permission denied", 1) + if f"as {words[0]}" in self.ignored: + return CommandResult("", "", 0) + assert all(p.startswith(a.home + "/") or p.startswith("/tmp/") for p in rest) + if words[0] == "cat": + return CommandResult(self.files.get(rest[0], ""), "", 0) + if words[0] == "mkdir": + return CommandResult("", "", 0) + if words[0] == "install": + assert words[1:3] == ["-m", "600"] + self.files[rest[1]] = self.files[rest[0]] + return CommandResult("", "", 0) + if words[0] == "rm": + for path in rest: + self.files.pop(path, None) + return CommandResult("", "", 0) + raise AssertionError(f"unexpected command as {name}: {words}") + + # -- the driver's own commands (stand-ins) ------------------------------------- + + def _mkuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + name, password = args[0], (args[1] if len(args) > 1 else None) + uid = 1001 + len(self.accounts) + self.accounts[name] = _Account(uid, f"/home/{name}", groups=[name], password=password) + return CommandResult("", "", 0) + + def _openuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + a = self.accounts[args[0]] + a.locked = False + if len(args) > 1: + a.password = args[1] + return CommandResult("", "", 0) + + def _lockuser(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + self.accounts[args[0]].locked = "lockuser" not in self.ignored + return CommandResult("", "", 0) + + def _islocked(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + return CommandResult("yes\n" if self.accounts[args[0]].locked else "no\n", "", 0) + + def _leavegroup(self, args: list[str], privileged: bool) -> CommandResult: + assert privileged + self.accounts[args[0]].groups.remove(args[1]) + return CommandResult("", "", 0) + + +class _Driver(LocalAccountsMixin): + ACCOUNT_SHELL = "/bin/sh" + ACCOUNT_ADMIN_GROUPS = ("wheel", "sudo") + SUDOERS_DIR = "/etc/sudoers.d" + ROOT_GROUP = "wheel" + + def __init__(self, host: _Host) -> None: + self.host = host + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: + return self.host.run(command, privileged, stdin) + + def _account_create(self, name: str, password_hash: str | None) -> list[list[str]]: + return [["mkuser", name, *([password_hash] if password_hash else [])]] + + def _account_open( + self, name: str, password_hash: str | None, *, locked: bool + ) -> list[list[str]]: + return [["openuser", name, *([password_hash] if password_hash else [])]] + + def _account_lock(self, name: str, *, locked: bool) -> list[list[str]]: + return [] if locked else [["lockuser", name]] + + def _account_locked(self, name: str, passwd: list[str]) -> bool: + result = self.run_command(shlex.join(["islocked", name]), privileged=True) + return result.stdout.strip() == "yes" + + def _account_leave_group(self, name: str, group: str, groups: list[str]) -> list[str]: + return ["leavegroup", name, group] + + +@pytest.fixture +def host() -> _Host: + return _Host() + + +@pytest.fixture +def driver(host: _Host) -> _Driver: + return _Driver(host) + + +def _root_commands(host: _Host) -> list[str]: + return [c for c, privileged in host.ran if privileged] + + +def _add_alice(host: _Host, *, locked: bool = False, keys: str = "") -> None: + host.accounts["alice"] = _Account( + 1001, "/home/alice", groups=["alice", "wheel"], locked=locked, password=HASH + ) + if keys: + host.files["/home/alice/.ssh/authorized_keys"] = keys + + +# -- the helpers --------------------------------------------------------------------- + + +def test_the_sudo_rule_names_the_account_and_asks_for_its_password() -> None: + assert sudoers_rule("alice") == "alice ALL=(ALL) ALL\n" + + +def test_the_rule_file_has_no_dot_sudo_would_skip() -> None: + # sudo ignores a file in an included directory whose name holds a dot. + path = sudoers_rule_path("/etc/sudoers.d", "a.lice") + assert path.startswith("/etc/sudoers.d/") + assert "." not in path.rsplit("/", 1)[1] + assert sudoers_rule_path("/etc/sudoers.d", "a.lice") != sudoers_rule_path( + "/etc/sudoers.d", "a_lice" + ) + + +def test_both_key_files_sshd_reads_by_default() -> None: + assert authorized_keys_paths("/home/alice") == ( + "/home/alice/.ssh/authorized_keys", + "/home/alice/.ssh/authorized_keys2", + ) + + +# -- reading -------------------------------------------------------------------------- + + +def test_an_account_is_read_with_its_groups_lock_sudo_and_keys( + host: _Host, driver: _Driver +) -> None: + _add_alice(host, keys=ALICE_KEY + "\n" + OWN_KEY + "\n") + host.other_rules.add("alice") + + [account] = driver.get_local_accounts(["alice"]) + + assert account == { + "name": "alice", + "uid": 1001, + "home": "/home/alice", + "shell": "/bin/sh", + "groups": ["alice", "wheel"], + "locked": False, + "admin": True, + "authorized_keys": [ALICE_KEY, OWN_KEY], + } + + +def test_an_account_that_is_not_there_is_left_out(driver: _Driver) -> None: + assert driver.get_local_accounts(["bob"]) == [] + + +def test_a_locked_accounts_keys_that_cannot_be_read_are_unknown( + host: _Host, driver: _Driver +) -> None: + _add_alice(host, locked=True, keys=ALICE_KEY + "\n") + + [account] = driver.get_local_accounts(["alice"]) + + assert account["locked"] is True + assert account["authorized_keys"] is None + + +def test_an_account_without_a_key_file_has_no_keys(host: _Host, driver: _Driver) -> None: + _add_alice(host) + + [account] = driver.get_local_accounts(["alice"]) + + assert account["authorized_keys"] == [] + + +def test_a_failing_read_raises_rather_than_report_nothing(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.failing.add("sudo -l") + + with pytest.raises(RuntimeError, match="alice"): + driver.get_local_accounts(["alice"]) + + +@pytest.mark.parametrize("name", ["", "Alice", "1alice", "al ice", "a;b", "-alice", "a" * 33]) +def test_a_name_no_guest_takes_is_refused(driver: _Driver, name: str) -> None: + with pytest.raises(ValueError, match="account name"): + driver.get_local_accounts([name]) + + +# -- adding --------------------------------------------------------------------------- + + +def test_a_new_account_gets_the_key_and_the_sudo_rule(host: _Host, driver: _Driver) -> None: + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["account", "sudo", "keys"] + assert host.accounts["alice"].password == HASH + assert host.files["/home/alice/.ssh/authorized_keys"] == ALICE_KEY + "\n" + assert host.files["/etc/sudoers.d/80-account-alice"] == "alice ALL=(ALL) ALL\n" + account = result["account"] + assert account is not None + assert account["admin"] is True and account["authorized_keys"] == [ALICE_KEY] + + +def test_every_command_root_runs_is_one_fixed_command(host: _Host, driver: _Driver) -> None: + driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + driver.lock_local_account("alice") + + for command in _root_commands(host): + assert not any(s in command for s in SHELL_SYNTAX), command + + +def test_the_staged_files_are_removed_again(host: _Host, driver: _Driver) -> None: + driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert host.staged == 2 + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_an_existing_account_keeps_its_own_keys(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys=OWN_KEY + "\n") + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.files["/home/alice/.ssh/authorized_keys"] == f"{OWN_KEY}\n{ALICE_KEY}\n" + + +def test_a_line_without_a_key_is_kept_and_not_taken_for_one(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys="# the laptop\n") + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.files["/home/alice/.ssh/authorized_keys"] == f"# the laptop\n{ALICE_KEY}\n" + + +def test_a_key_already_there_is_not_written_again(host: _Host, driver: _Driver) -> None: + # The same key with options in front and another comment is the same key. + _add_alice(host, keys=f'from="10.0.0.0/8" {ALICE_KEY.rsplit(" ", 1)[0]} old\n') + + result = driver.add_local_account("alice", ALICE_KEY) + + assert result["success"], result["output"] + assert host.staged == 0 + assert "already" in result["steps"][-1]["output"] + + +def test_a_locked_account_is_opened_again(host: _Host, driver: _Driver) -> None: + _add_alice(host, locked=True) + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH) + + assert result["success"], result["output"] + assert host.accounts["alice"].locked is False + assert result["account"] is not None and result["account"]["locked"] is False + + +def test_without_sudo_the_rule_and_the_admin_groups_go(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.files["/etc/sudoers.d/80-account-alice"] = sudoers_rule("alice") + + result = driver.add_local_account("alice", ALICE_KEY, admin=False) + + assert result["success"], result["output"] + assert "/etc/sudoers.d/80-account-alice" not in host.files + assert host.accounts["alice"].groups == ["alice"] + + +def test_sudo_another_rule_still_grants_fails_the_step(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.other_rules.add("alice") + + result = driver.add_local_account("alice", ALICE_KEY, admin=False) + + assert not result["success"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "another rule" in sudo["output"] + # The key still went in: the steps after the account are independent. + assert host.files["/home/alice/.ssh/authorized_keys"] == ALICE_KEY + "\n" + + +def test_a_rule_sudo_does_not_read_fails_the_step(host: _Host, driver: _Driver) -> None: + host.includes_dropins = False + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "/etc/sudoers.d" in sudo["output"] + + +def test_a_rule_visudo_rejects_is_not_installed(host: _Host, driver: _Driver) -> None: + host.failing.add("visudo") + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + assert not result["success"] + assert "/etc/sudoers.d/80-account-alice" not in host.files + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_a_failed_create_stops_before_sudo_and_keys(host: _Host, driver: _Driver) -> None: + host.failing.add("mkuser") + + result = driver.add_local_account("alice", ALICE_KEY, admin=True) + + assert not result["success"] + assert [s["step"] for s in result["steps"]] == ["account"] + assert result["account"] is None + + +@pytest.mark.parametrize( + "key", + ["", "not a key", ALICE_KEY + "\n" + OWN_KEY, "ssh-ed25519", "ssh-ed25519 AAAA;rm"], +) +def test_a_public_key_that_is_none_is_refused(driver: _Driver, key: str) -> None: + with pytest.raises(ValueError, match="public key"): + driver.add_local_account("alice", key) + + +@pytest.mark.parametrize("password_hash", ["secret", "$6$salt$ab cd", "$2b$10$" + "x" * 53]) +def test_a_password_hash_the_guest_does_not_take_is_refused( + driver: _Driver, password_hash: str +) -> None: + # This driver takes SHA-512 crypt: neither plain text nor bcrypt. + with pytest.raises(ValueError, match="password hash"): + driver.add_local_account("alice", ALICE_KEY, password_hash=password_hash) + + +def test_the_login_users_own_account_is_never_touched(host: _Host, driver: _Driver) -> None: + for change in ( + lambda: driver.add_local_account("netadmin", ALICE_KEY), + lambda: driver.lock_local_account("netadmin"), + ): + with pytest.raises(ValueError, match="logs in"): + change() + assert _root_commands(host) == [] + + +def test_nothing_changes_when_the_login_user_cannot_be_told(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.login = "" + + with pytest.raises(RuntimeError, match="logs in"): + driver.lock_local_account("alice") + assert _root_commands(host) == [] + + +def test_a_system_account_is_never_touched(host: _Host, driver: _Driver) -> None: + with pytest.raises(ValueError, match="system account"): + driver.lock_local_account("daemon") + assert _root_commands(host) == [] + + +# -- locking -------------------------------------------------------------------------- + + +def test_locking_takes_keys_sudo_and_login_away(host: _Host, driver: _Driver) -> None: + _add_alice(host, keys=ALICE_KEY + "\n" + OWN_KEY + "\n") + host.files["/home/alice/.ssh/authorized_keys2"] = OWN_KEY + "\n" + host.files["/etc/sudoers.d/80-account-alice"] = sudoers_rule("alice") + + result = driver.lock_local_account("alice") + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["keys", "lock", "sudo"] + a = host.accounts["alice"] + assert a.locked and a.groups == ["alice"] and a.password == HASH + assert not [p for p in host.files if p.startswith("/home/alice/")] + assert "/etc/sudoers.d/80-account-alice" not in host.files + account = result["account"] + assert account is not None and account["locked"] and not account["admin"] + + +def test_a_rule_left_elsewhere_is_named_but_harmless_once_locked( + host: _Host, driver: _Driver +) -> None: + _add_alice(host) + host.other_rules.add("alice") + + result = driver.lock_local_account("alice") + + assert result["success"], result["output"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert "another rule" in sudo["output"] and "locked" in sudo["output"] + + +def test_the_lock_comes_even_when_the_keys_could_not_go(host: _Host, driver: _Driver) -> None: + _add_alice(host, locked=True, keys=ALICE_KEY + "\n") + + result = driver.lock_local_account("alice") + + assert not result["success"] + keys = next(s for s in result["steps"] if s["step"] == "keys") + assert not keys["ok"] + assert next(s for s in result["steps"] if s["step"] == "lock")["ok"] + assert host.accounts["alice"].locked + + +def test_locking_an_account_that_is_not_there_fails(driver: _Driver) -> None: + result = driver.lock_local_account("bob") + + assert not result["success"] + assert "no account" in result["output"] + assert result["account"] is None + + +# -- what fails is said --------------------------------------------------------------- + + +@pytest.mark.parametrize("failing", ["getent", "id -Gn"]) +def test_an_account_that_cannot_be_read_raises(host: _Host, driver: _Driver, failing: str) -> None: + _add_alice(host) + host.failing.add(failing) + + with pytest.raises(RuntimeError, match="alice"): + driver.get_local_accounts(["alice"]) + + +@pytest.mark.parametrize( + ("failing", "step", "said"), + [ + ("rm -f", "sudo", "80-account-alice"), + ("leavegroup", "sudo", "out of wheel"), + ("as cat", "keys", "cannot read"), + ("as install", "keys", "authorized_keys"), + ("mktemp", "keys", "cannot stage"), + ], +) +def test_a_step_that_fails_says_why( + host: _Host, driver: _Driver, failing: str, step: str, said: str +) -> None: + _add_alice(host, keys=OWN_KEY + "\n") + host.failing.add(failing) + + result = driver.add_local_account("alice", ALICE_KEY) + + failed = next(s for s in result["steps"] if s["step"] == step) + assert not failed["ok"] and said in failed["output"] + assert not result["success"] + + +def test_a_rule_that_cannot_be_installed_fails_the_step(host: _Host, driver: _Driver) -> None: + host.failing.add("install -m") + + result = driver.add_local_account("alice", ALICE_KEY, password_hash=HASH, admin=True) + + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "80-account-alice" in sudo["output"] + assert not [p for p in host.files if p.startswith("/tmp/")] + + +@pytest.mark.parametrize( + ("failing", "ignored", "step", "said"), + [ + ({"lockuser"}, set(), "lock", "lockuser: failed"), + (set(), {"lockuser"}, "lock", "still not locked"), + ({"as rm"}, set(), "keys", "cannot remove"), + (set(), {"as rm"}, "keys", "still has keys"), + ], +) +def test_a_lock_that_did_not_take_says_so( + host: _Host, driver: _Driver, failing: set[str], ignored: set[str], step: str, said: str +) -> None: + _add_alice(host, keys=ALICE_KEY + "\n") + host.failing |= failing + host.ignored |= ignored + + result = driver.lock_local_account("alice") + + failed = next(s for s in result["steps"] if s["step"] == step) + assert not failed["ok"] and said in failed["output"] + assert not result["success"] + + +def test_an_account_that_cannot_be_read_back_is_said(host: _Host, driver: _Driver) -> None: + _add_alice(host) + host.failing.add("sudo -l") + + result = driver.lock_local_account("alice") + + assert result["account"] is None + assert "[read back] cannot ask sudo about alice" in result["output"] + + +# -- a key store of the driver's own --------------------------------------------------- + + +class _OwnKeyStore(_Driver): + """A driver whose host keeps keys in a place of its own.""" + + def __init__(self, host: _Host) -> None: + super().__init__(host) + self.store: dict[str, list[str]] = {} + + def _account_read_keys(self, name: str, home: str) -> list[str] | None: + return list(self.store.get(name, [])) + + def _account_add_key(self, name: str, home: str, public_key: str) -> FixStepOutcomeDict: + self.store.setdefault(name, []).append(f"{home} {public_key}") + return {"ok": True, "output": "stored"} + + def _account_remove_keys(self, name: str, home: str) -> FixStepOutcomeDict: + self.store.pop(name, None) + return {"ok": True, "output": "dropped"} + + +def test_a_driver_can_keep_the_keys_elsewhere(host: _Host) -> None: + driver = _OwnKeyStore(host) + + added = driver.add_local_account("alice", ALICE_KEY) + assert added["success"], added["output"] + assert driver.store == {"alice": [f"/home/alice {ALICE_KEY}"]} + assert added["account"] is not None + assert added["account"]["authorized_keys"] == [f"/home/alice {ALICE_KEY}"] + assert not [p for p in host.files if p.startswith("/home/")] + + locked = driver.lock_local_account("alice") + assert locked["success"], locked["output"] + assert driver.store == {} + + +def test_a_key_file_the_host_adds_is_read_too(host: _Host) -> None: + class _ExtraFile(_Driver): + def _account_key_files(self, name: str, home: str) -> tuple[str, ...]: + return (*super()._account_key_files(name, home), f"/home/{name}/extra") + + _add_alice(host, keys=OWN_KEY + "\n") + host.files["/home/alice/extra"] = ALICE_KEY + "\n" + + [account] = _ExtraFile(host).get_local_accounts(["alice"]) + + assert account["authorized_keys"] == [OWN_KEY, ALICE_KEY] diff --git a/tests/test_network_target_uplinks.py b/tests/test_network_target_uplinks.py new file mode 100644 index 0000000..bfc045b --- /dev/null +++ b/tests/test_network_target_uplinks.py @@ -0,0 +1,25 @@ +"""A network target can say which physical NICs it leaves the host through. + +A VM on a VLAN-aware bridge without a VLAN tag only gets an address when the +switch port behind the bridge carries an untagged network. To look that port up, +the caller needs the bridge's physical NICs: their LLDP neighbours name the +switch and the port (NetOrk/netork#1325). They come in groups, one per way out +of the bridge: a bond's slaves together, a single NIC alone. A bond has no +neighbour of its own, and its slaves are one link, so one slave's neighbour +speaks for the others when they report none. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types.models import NetworkTargetDict + + +def test_uplinks_is_an_optional_list_of_nic_groups(): + assert "uplinks" in NetworkTargetDict.__optional_keys__ + assert get_type_hints(NetworkTargetDict)["uplinks"] == list[list[str]] + + +def test_contract_core_stays_required(): + assert {"name", "kind", "vlan_aware"} <= NetworkTargetDict.__required_keys__ diff --git a/tests/test_package_updates.py b/tests/test_package_updates.py index 63d1342..b12a796 100644 --- a/tests/test_package_updates.py +++ b/tests/test_package_updates.py @@ -176,6 +176,18 @@ __DNF_RC=0 with pytest.raises(ValueError, match="exit status 1"): parse_dnf_security(out) + def test_the_command_reads_only_the_cached_metadata(self, tmp_path): + """An expired cache made dnf download the repositories' metadata inside the + poll, which outlasted its whole budget (#76).""" + args = tmp_path / "args" + dnf = tmp_path / "dnf" + dnf.write_text(f"#!/bin/sh\nprintf '%s\\n' \"$@\" > {args}\n") + dnf.chmod(0o755) + env = {"PATH": f"{tmp_path}:{os.environ.get('PATH', '/usr/bin:/bin')}"} + subprocess.run(["sh", "-c", DNF_SECURITY_COMMAND], capture_output=True, env=env) + + assert "--cacheonly" in args.read_text().split() + @pytest.mark.parametrize( ("nevra", "name"), [ diff --git a/tests/test_ping_sweep.py b/tests/test_ping_sweep.py index 90bef7d..03a34db 100644 --- a/tests/test_ping_sweep.py +++ b/tests/test_ping_sweep.py @@ -5,7 +5,6 @@ from napalm.base import NetworkDriver from napalm_device_types import DeviceTypeDriver, PingSweepMixin, driver_supports_ping - # ── Fakes ───────────────────────────────────────────────────────────────────── @@ -225,4 +224,9 @@ def test_ping_sweep_stops_when_stop_requested(): def test_a_malformed_success_is_no_answer_not_an_abort(success): entry = PingSweepMixin._parse_ping_reply("192.0.2.1", {"success": success}) - assert entry == {"ip": "192.0.2.1", "alive": False, "rtt_ms": None, "error": "malformed ping reply"} + assert entry == { + "ip": "192.0.2.1", + "alive": False, + "rtt_ms": None, + "error": "malformed ping reply", + } diff --git a/tests/test_port_forwards.py b/tests/test_port_forwards.py index 7807d39..19b3ccb 100644 --- a/tests/test_port_forwards.py +++ b/tests/test_port_forwards.py @@ -19,7 +19,7 @@ from napalm_device_types.nat_vpn import NatVpnMixin def test_a_firewall_and_a_gateway_share_the_declaration(): assert issubclass(FirewallDriver, NatVpnMixin) assert issubclass(ResidentialGatewayDriver, NatVpnMixin) - assert "def get_port_forwards(self) -> List[PortForwardDict]" in inspect.getsource(NatVpnMixin) + assert "def get_port_forwards(self) -> list[PortForwardDict]" in inspect.getsource(NatVpnMixin) def test_it_is_declared_once(): diff --git a/tests/test_provisioning.py b/tests/test_provisioning.py index dd52b73..56ff10f 100644 --- a/tests/test_provisioning.py +++ b/tests/test_provisioning.py @@ -4,6 +4,7 @@ the same for every hypervisor (NetOrk/netork#794).""" from __future__ import annotations import pytest + from napalm_device_types import HypervisorDriver from napalm_device_types.provisioning import ( QEMU_GUEST_AGENTS, diff --git a/tests/test_public_contracts_1113.py b/tests/test_public_contracts_1113.py new file mode 100644 index 0000000..5a861b5 --- /dev/null +++ b/tests/test_public_contracts_1113.py @@ -0,0 +1,47 @@ +"""Public contracts for what an application reached through driver internals. + +netOrk followed a firewall's upgrade, edited its resolver's forwards and read +its OSPF state through the driver's private REST helpers (NetOrk/netork#1113). +Each becomes a declared method with a neutral shape; the vendor's own format +is translated by the driver. +""" + +from __future__ import annotations + +from typing import get_type_hints + +from napalm_device_types import ( + DnsForwardDict, + DnsForwardMixin, + OspfDict, + OspfInterfaceDict, + OspfMixin, + UpdateMixin, + UpdateProgressDict, +) + + +def test_update_progress_is_a_state_and_its_log(): + assert set(get_type_hints(UpdateProgressDict)) == {"state", "log"} + assert not hasattr(UpdateMixin, "get_update_progress") + + +def test_a_dns_forward_names_the_domain_and_where_it_goes(): + assert set(get_type_hints(DnsForwardDict)) == {"domain", "server", "port", "enabled"} + for name in ("get_dns_forwards", "set_dns_forward", "remove_dns_forward"): + assert not hasattr(DnsForwardMixin, name), name + + +def test_ospf_in_neutral_terms(): + assert set(get_type_hints(OspfDict)) == {"router_id", "areas", "interfaces"} + assert set(get_type_hints(OspfInterfaceDict)) == { + "address", + "prefix_length", + "area", + "network_type", + "authentication", + "neighbors", + "neighbors_adjacent", + "cost", + } + assert not hasattr(OspfMixin, "get_ospf") diff --git a/tests/test_role_contracts.py b/tests/test_role_contracts.py index a8b9f50..0c60958 100644 --- a/tests/test_role_contracts.py +++ b/tests/test_role_contracts.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """The contract that makes multi-role drivers safe. A role base declares what a device of that kind can be asked for. It must not diff --git a/tests/test_send_wake_on_lan.py b/tests/test_send_wake_on_lan.py index b392ffd..128855c 100644 --- a/tests/test_send_wake_on_lan.py +++ b/tests/test_send_wake_on_lan.py @@ -9,6 +9,7 @@ sibling base. """ import pytest + from napalm_device_types import FirewallDriver diff --git a/tests/test_service_account.py b/tests/test_service_account.py new file mode 100644 index 0000000..9bc270a --- /dev/null +++ b/tests/test_service_account.py @@ -0,0 +1,301 @@ +"""netOrk's own account on a host: created through the account it logs in with today. + +The fake host of :mod:`tests.test_local_accounts`, taught the rule that lets +an account run everything as root without a password. +""" + +from __future__ import annotations + +import shlex + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.local_accounts import ( + service_sudoers_rule, + service_sudoers_rule_path, + sudoers_rule_path, +) +from tests.test_local_accounts import ( + OWN_KEY, + SHELL_SYNTAX, + _Account, + _Driver, + _Host, + _root_commands, +) + +NETORK_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINetork netork@central" + + +class _ServiceHost(_Host): + """The fake host, whose sudo also takes the rule without a password.""" + + def is_passwordless(self, name: str) -> bool: + rule = self.files.get(service_sudoers_rule_path(_Driver.SUDOERS_DIR, name)) + return self.includes_dropins and rule is not None + + def _sudo(self, args: list[str], privileged: bool) -> CommandResult: + name = args[2] + if not self.is_passwordless(name): + return super()._sudo(args, privileged) + out = f"User {name} may run the following commands on host:\n (ALL) NOPASSWD: ALL\n" + return CommandResult(out, "", 0) + + def _visudo(self, args: list[str], privileged: bool) -> CommandResult: + content = self.files[args[1]] + if content.endswith(" ALL=(ALL) NOPASSWD: ALL\n") and "\n" not in content[:-1]: + return CommandResult("parsed OK", "", 0) + return super()._visudo(args, privileged) + + +@pytest.fixture +def host() -> _ServiceHost: + return _ServiceHost() + + +@pytest.fixture +def driver(host: _ServiceHost) -> _Driver: + return _Driver(host) + + +def test_the_rule_runs_everything_as_root_without_a_password() -> None: + assert service_sudoers_rule("netork") == "netork ALL=(ALL) NOPASSWD: ALL\n" + + +def test_the_rule_has_a_file_of_its_own_no_personal_rule_shares() -> None: + path = service_sudoers_rule_path("/etc/sudoers.d", "net.ork_1") + assert path == "/etc/sudoers.d/85-service-net_2eork__1" + assert path != sudoers_rule_path("/etc/sudoers.d", "net.ork_1") + + +def test_a_new_service_account_gets_the_key_and_sudo_without_a_password( + host: _ServiceHost, driver: _Driver +) -> None: + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert [s["step"] for s in result["steps"]] == ["account", "sudo", "keys"] + assert host.accounts["netork"].password is None + assert host.files["/home/netork/.ssh/authorized_keys"] == NETORK_KEY + "\n" + assert host.is_passwordless("netork") + account = result["account"] + assert account is not None and account["admin"] + + +def test_every_command_root_runs_is_one_fixed_command(host: _ServiceHost, driver: _Driver) -> None: + driver.add_service_account("netork", NETORK_KEY) + + for command in _root_commands(host): + assert not any(s in command for s in SHELL_SYNTAX), command + + +def test_an_existing_account_keeps_its_keys_and_gets_the_rule( + host: _ServiceHost, driver: _Driver +) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + host.files["/home/netork/.ssh/authorized_keys"] = OWN_KEY + "\n" + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert host.files["/home/netork/.ssh/authorized_keys"] == f"{OWN_KEY}\n{NETORK_KEY}\n" + assert host.is_passwordless("netork") + + +def test_an_existing_system_account_may_become_netorks(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["netork"] = _Account(998, "/var/lib/netork", groups=["netork"]) + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + + +def test_a_locked_account_is_opened_again(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"], locked=True) + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert not host.accounts["netork"].locked + + +def test_the_account_netork_logs_in_with_already_only_gets_what_it_lacks( + host: _ServiceHost, driver: _Driver +) -> None: + host.login = "netork" + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + host.files["/home/netork/.ssh/authorized_keys"] = NETORK_KEY + "\n" + + result = driver.add_service_account("netork", NETORK_KEY) + + assert result["success"], result["output"] + assert not any(c.startswith("openuser") for c in _root_commands(host)) + assert "already" in result["output"] + + +def test_root_is_never_netorks_account(host: _ServiceHost, driver: _Driver) -> None: + host.accounts["toor"] = _Account(0, "/root", groups=["wheel"]) + + with pytest.raises(ValueError, match="root"): + driver.add_service_account("toor", NETORK_KEY) + assert not _root_commands(host) + + +@pytest.mark.parametrize("name", ["Netork", "net ork", "-n", ""]) +def test_a_name_no_host_takes_is_refused(driver: _Driver, name: str) -> None: + with pytest.raises(ValueError): + driver.add_service_account(name, NETORK_KEY) + + +def test_a_key_that_is_none_is_refused(driver: _Driver) -> None: + with pytest.raises(ValueError): + driver.add_service_account("netork", "not a key") + + +def test_a_rule_sudo_does_not_read_fails_the_step(host: _ServiceHost, driver: _Driver) -> None: + host.includes_dropins = False + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + sudo = next(s for s in result["steps"] if s["step"] == "sudo") + assert not sudo["ok"] and "does not read it" in sudo["output"] + + +def test_a_rule_with_a_password_is_not_enough(host: _ServiceHost, driver: _Driver) -> None: + host.other_rules.add("netork") + host.includes_dropins = False + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + + +def test_a_failed_create_stops_before_sudo_and_keys(host: _ServiceHost, driver: _Driver) -> None: + host.failing.add("mkuser") + + result = driver.add_service_account("netork", NETORK_KEY) + + assert not result["success"] + assert [s["step"] for s in result["steps"]] == ["account"] + assert result["account"] is None + + +def test_the_staged_files_are_removed_again(host: _ServiceHost, driver: _Driver) -> None: + driver.add_service_account("netork", NETORK_KEY) + + assert not [p for p in host.files if p.startswith("/tmp/")] + + +def test_a_personal_account_of_the_same_name_keeps_its_own_rule_apart( + host: _ServiceHost, driver: _Driver +) -> None: + driver.add_service_account("netork", NETORK_KEY) + + rules = [c for c in _root_commands(host) if c.startswith("install")] + assert [shlex.split(c)[-1] for c in rules] == [ + service_sudoers_rule_path(_Driver.SUDOERS_DIR, "netork") + ] + + +# -- a fixed uid (infrastructure BAUSTANDARD A) ------------------------------------------ + + +class _UidDriver(_Driver): + def _account_create_with_uid(self, name: str, uid: int) -> list[list[str]]: + return [["mkuseruid", name, str(uid)]] + + +def _mkuseruid(host: _ServiceHost): + def handler(args: list[str], privileged: bool) -> CommandResult: + assert privileged + name, uid = args[0], int(args[1]) + host.accounts[name] = _Account(uid, f"/home/{name}", groups=[name]) + return CommandResult("", "", 0) + + return handler + + +def _getent_uid(host: _ServiceHost, original): + def handler(args: list[str], privileged: bool) -> CommandResult: + if args[1].isdigit(): + owner = next((n for n, a in host.accounts.items() if a.uid == int(args[1])), None) + if owner is None: + return CommandResult("", "", 2) + a = host.accounts[owner] + return CommandResult(f"{owner}:*:{a.uid}:{a.uid}::{a.home}:{a.shell}\n", "", 0) + return original(args, privileged) + + return handler + + +@pytest.fixture +def uid_driver(host: _ServiceHost) -> _UidDriver: + host._mkuseruid = _mkuseruid(host) # type: ignore[attr-defined] + host._getent = _getent_uid(host, host._getent) # type: ignore[method-assign] + return _UidDriver(host) + + +def test_a_new_account_gets_the_uid_asked_for(host: _ServiceHost, uid_driver: _UidDriver) -> None: + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert result["success"], result["output"] + assert host.accounts["netork"].uid == 900 + account = result["account"] + assert account is not None and account["uid"] == 900 + + +def test_a_uid_another_account_has_stops_before_anything_is_made( + host: _ServiceHost, uid_driver: _UidDriver +) -> None: + host.accounts["fwupd-refresh"] = _Account(900, "/run/fwupd") + + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert not result["success"] + assert "fwupd-refresh" in result["output"] + assert "netork" not in host.accounts + + +def test_an_existing_account_keeps_its_uid_and_says_so( + host: _ServiceHost, uid_driver: _UidDriver +) -> None: + host.accounts["netork"] = _Account(1001, "/home/netork", groups=["netork"]) + + result = uid_driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert result["success"], result["output"] + assert host.accounts["netork"].uid == 1001 + assert "uid 1001" in result["output"] + + +def test_a_uid_root_has_is_refused(uid_driver: _UidDriver) -> None: + with pytest.raises(ValueError): + uid_driver.add_service_account("netork", NETORK_KEY, uid=0) + + +def test_a_driver_that_cannot_set_a_uid_says_so(host: _ServiceHost, driver: _Driver) -> None: + result = driver.add_service_account("netork", NETORK_KEY, uid=900) + + assert not result["success"] + assert "uid" in result["output"] + + +# -- another service's account of the same name (bw-01: the satellite's) ------------------ + + +@pytest.mark.parametrize( + "shell", ["/usr/sbin/nologin", "/sbin/nologin", "/bin/false", "/usr/bin/false"] +) +def test_an_account_that_cannot_log_in_is_another_services_and_left_alone( + host: _ServiceHost, driver: _Driver, shell: str +) -> None: + host.accounts["netork"] = _Account( + 999, "/home/netork", shell=shell, groups=["netork", "docker"] + ) + + with pytest.raises(ValueError, match="cannot log in"): + driver.add_service_account("netork", NETORK_KEY) + assert not _root_commands(host) + assert service_sudoers_rule_path(_Driver.SUDOERS_DIR, "netork") not in host.files diff --git a/tests/test_service_clusters.py b/tests/test_service_clusters.py new file mode 100644 index 0000000..9c74810 --- /dev/null +++ b/tests/test_service_clusters.py @@ -0,0 +1,596 @@ +"""get_service_clusters: the configuration that says which services run together. + +Two hosts that answer at one virtual IP may run the same service by accident. +What proves a cluster is the configuration the members share: Patroni's scope +and the members its DCS lists, adguardhome-sync's origin and replicas, the +address a BunkerWeb worker takes its configuration from. Reading that is the +same on every Linux host, so the command and its parse live here once, and a +driver only carries the command across (netOrk #1125). + +The same configuration holds secrets -- Patroni's superuser password, the sync's +AdGuard logins, a database URI -- so the command filters on the host: only +allowlisted fields ever leave it. The tests below run the command for real to +hold it to that. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from napalm_device_types.service_clusters import ( + SERVICE_CLUSTERS_COMMAND, + ServiceClustersMixin, + parse_service_clusters, + service_clusters_command, +) + +SECRET = "s3cr3t-do-not-leak" + + +def _wire(*sections: str, systemd: bool = True, docker: str = "[docker]", noise: str = "") -> str: + head = "[systemd]" if systemd else "[no-systemd]" + body = ( + "\n".join([head, *sections[:1], docker, *sections[1:]]) if sections else f"{head}\n{docker}" + ) + return f"{noise}SVCC_BEGIN\n{body}\nSVCC_END\n" + + +DB01_CLUSTER = { + "members": [ + { + "name": "db-01", + "role": "leader", + "state": "running", + "host": "10.7.234.12", + "port": 5432, + }, + { + "name": "db-02", + "role": "sync_standby", + "state": "streaming", + "host": "10.7.234.11", + "port": 5432, + "lag": 0, + }, + ], + "scope": "netork-db", +} + + +def _patroni(*records: str, unit: str = "patroni.service") -> str: + return "\n".join([f"[patroni {unit}]", *records]) + + +def _container(name: str, image: str, *env: str) -> str: + return "\n".join([f"[container {name}]", f"image {image}", *(f"env {e}" for e in env)]) + + +class TestPatroni: + def test_the_members_its_dcs_lists_with_their_roles(self): + reading = parse_service_clusters( + _wire( + _patroni( + "active active", + "config /etc/patroni/config.yml", + "scope netork-db", + "name db-01", + "restapi 10.7.234.12:8008", + "rest " + json.dumps(DB01_CLUSTER), + ) + ) + ) + + assert reading["patroni"] == [ + { + "mechanism": "patroni", + "service": "patroni", + "source": "patroni", + "cluster": "netork-db", + "running": True, + "members": [ + { + "name": "db-01", + "address": "10.7.234.12", + "port": 5432, + "role": "leader", + "state": "running", + "this_host": True, + "config_from": None, + }, + { + "name": "db-02", + "address": "10.7.234.11", + "port": 5432, + "role": "sync_standby", + "state": "streaming", + "this_host": False, + "config_from": None, + }, + ], + } + ] + + def test_without_its_rest_api_it_is_itself_in_its_scope(self): + reading = parse_service_clusters( + _wire( + _patroni( + "active active", + "scope 'netork-db' # the cluster", + "name db-01", + "restapi 10.7.234.12:8008", + "rest-unreachable", + ) + ) + ) + + (cluster,) = reading["patroni"] + assert cluster["cluster"] == "netork-db" + assert cluster["members"] == [ + { + "name": "db-01", + "address": "10.7.234.12", + "port": None, + "role": None, + "state": None, + "this_host": True, + "config_from": None, + } + ] + + def test_an_unreadable_config_is_not_looked_at(self): + reading = parse_service_clusters( + _wire(_patroni("active active", "unreadable /etc/patroni/config.yml")) + ) + + assert reading["patroni"] is None + + def test_a_config_without_a_scope_declares_no_cluster(self): + assert ( + parse_service_clusters(_wire(_patroni("active active", "name db-01")))["patroni"] == [] + ) + + def test_a_host_without_systemd_is_not_looked_at(self): + assert parse_service_clusters(_wire(systemd=False))["patroni"] is None + + def test_no_patroni_is_looked_and_found_nothing(self): + assert parse_service_clusters(_wire())["patroni"] == [] + + +class TestAdguardhomeSync: + def test_the_origin_pushes_to_each_replica(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "adguardhome-sync", + "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL http://10.9.100.5:3000", + "REPLICA1_URL http://10.9.100.6:3000", + "REPLICA2_URL https://dns-03.example.org", + ), + ) + ) + + (cluster,) = reading["adguardhome_sync"] + assert (cluster["mechanism"], cluster["service"], cluster["cluster"]) == ( + "adguardhome-sync", + "adguardhome", + "10.9.100.5:3000", + ) + assert [ + (m["address"], m["port"], m["role"], m["config_from"]) for m in cluster["members"] + ] == [ + ("10.9.100.5", 3000, "origin", None), + ("10.9.100.6", 3000, "replica", 0), + ("dns-03.example.org", 443, "replica", 0), + ] + assert not any(m["this_host"] for m in cluster["members"]) + + def test_a_single_replica_url_counts(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "sync", + "bakito/adguardhome-sync", + "ORIGIN_URL http://a:3000", + "REPLICA_URL http://b:3000", + ), + ) + ) + + assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == ["a", "b"] + + def test_a_sync_configured_by_file_is_not_looked_at(self): + """Its YAML is not read: the cluster is there, unseen.""" + reading = parse_service_clusters(_wire("", _container("sync", "bakito/adguardhome-sync"))) + + assert reading["adguardhome_sync"] is None + + +class TestBunkerweb: + def test_a_worker_takes_its_configuration_from_the_manager_it_admits(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "bunkerweb", "bunkerity/bunkerweb@sha256:2026", "API_WHITELIST_IP 10.50.100.5" + ), + ) + ) + + (cluster,) = reading["bunkerweb"] + assert (cluster["service"], cluster["cluster"]) == ("bunkerweb", "10.50.100.5") + assert cluster["members"] == [ + { + "name": None, + "address": None, + "port": 5000, + "role": "worker", + "state": None, + "this_host": True, + "config_from": 1, + }, + { + "name": None, + "address": "10.50.100.5", + "port": None, + "role": "manager", + "state": None, + "this_host": False, + "config_from": None, + }, + ] + + def test_the_default_whitelist_names_no_manager(self): + """127.0.0.0/8 and a docker network are BunkerWeb's default, not a peer.""" + reading = parse_service_clusters( + _wire( + "", + _container( + "bw", "bunkerity/bunkerweb:1.6", "API_WHITELIST_IP 127.0.0.0/8 10.20.30.0/24" + ), + ) + ) + + assert reading["bunkerweb"] == [] + + def test_a_manager_names_its_instances(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "bw", + "bunkerity/bunkerweb-all-in-one:1.6", + "BUNKERWEB_INSTANCES 10.50.100.6:5000 bw-03", + ), + ), + ) + + (cluster,) = reading["bunkerweb"] + assert cluster["cluster"] is None # the manager cannot name its own address + assert [ + (m["address"], m["port"], m["role"], m["this_host"], m["config_from"]) + for m in cluster["members"] + ] == [ + (None, None, "manager", True, None), + ("10.50.100.6", 5000, "worker", False, 0), + ("bw-03", 5000, "worker", False, 0), + ] + + def test_a_manager_without_instances_is_no_cluster(self): + """proxy-01's all-in-one keeps its instances in its database.""" + reading = parse_service_clusters( + _wire("", _container("bw", "bunkerity/bunkerweb-all-in-one@sha256:2b34")) + ) + + assert reading["bunkerweb"] == [] + + def test_its_ui_and_scheduler_side_containers_are_not_instances(self): + reading = parse_service_clusters( + _wire("", _container("ui", "bunkerity/bunkerweb-ui:1.6", "API_WHITELIST_IP 10.0.0.9")) + ) + + assert reading["bunkerweb"] == [] + + +class TestTheReport: + def test_a_host_without_docker_runs_no_containers(self): + reading = parse_service_clusters(_wire(docker="[no-docker]")) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], []) + + def test_a_docker_it_may_not_ask_is_not_looked_at(self): + reading = parse_service_clusters(_wire(docker="[docker-unreadable]")) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == (None, None) + + def test_noise_before_the_report_is_ignored(self): + assert parse_service_clusters(_wire(noise="Last login: today\n"))["patroni"] == [] + + def test_a_report_cut_short_raises_without_quoting_it(self): + with pytest.raises(ValueError) as caught: + parse_service_clusters("SVCC_BEGIN\n[systemd]\nscope netork-db\n") + + assert "netork-db" not in str(caught.value) + + +# --------------------------------------------------------------------------- +# The command itself, run against stubs and real files +# --------------------------------------------------------------------------- + +PATRONI_YML = f"""scope: netork-db +namespace: /service/ +name: db-01 + +restapi: + listen: 10.7.234.12:8008 + connect_address: 10.7.234.12:8008 + authentication: + username: patroni + password: {SECRET} + +etcd3: + hosts: + - 10.7.234.12:2379 + - 10.7.234.13:2379 + password: {SECRET} + +postgresql: + connect_address: 10.7.234.12:5432 + authentication: + superuser: + username: postgres + password: {SECRET} +tags: + name: not-the-member-name +""" + + +def _awks() -> list: + found = [] + for name, argv in (("mawk", ["mawk"]), ("busybox", ["busybox", "awk"]), ("gawk", ["gawk"])): + if shutil.which(argv[0]): + found.append(pytest.param(argv, id=name)) + return found or [pytest.param(None, marks=pytest.mark.skip(reason="no awk to run"))] + + +class _Host: + """A host as far as the command can tell: units, containers, a REST API, files.""" + + def __init__(self, root: Path, awk: list) -> None: + self.root = root + self.bin = root / "bin" + self.bin.mkdir() + self.state = root / "state" + self.state.mkdir() + self.booted = root / "run-systemd-system" + self.booted.mkdir() + (self.state / "units").write_text("") + (self.state / "containers").write_text("") + cat = shutil.which("cat") + self._stub( + "systemctl", + 'case "$1" in\n' + f' list-units) {cat} "$d/units" ;;\n' + f' show) for last; do :; done; {cat} "$d/show-$last" ;;\n' + "esac\n", + ) + self._stub( + "docker", + 'case "$1" in\n' + f' ps) {cat} "$d/containers" ;;\n' + " inspect) for last; do :; done;\n" + ' case "$3" in *Config.Image*) ' + f'{cat} "$d/image-$last" ;; *) {cat} "$d/env-$last" ;; esac ;;\n' + "esac\n", + ) + self._stub("curl", f'for last; do :; done; {cat} "$d/rest" 2>/dev/null || exit 7\n') + for tool in ("sed", "head", "tr"): + if shutil.which(tool): + (self.bin / tool).symlink_to(shutil.which(tool)) + wrapper = self.bin / "awk" + program = " ".join([shutil.which(awk[0]) or awk[0], *awk[1:]]) + wrapper.write_text(f'#!/bin/sh\nexec {program} "$@"\n') + wrapper.chmod(0o755) + + def _stub(self, name: str, body: str) -> None: + stub = self.bin / name + stub.write_text(f"#!/bin/sh\nd={self.state}\n{body}") + stub.chmod(0o755) + + def unit(self, name: str, *properties: str) -> None: + with (self.state / "units").open("a") as units: + units.write(f"{name} loaded active running {name}\n") + (self.state / f"show-{name}").write_text("".join(f"{p}\n" for p in properties)) + + def container(self, name: str, image: str, *env: str) -> None: + with (self.state / "containers").open("a") as containers: + containers.write(f"{name}\n") + (self.state / f"image-{name}").write_text(f"{image}\n") + (self.state / f"env-{name}").write_text("".join(f"{e}\n" for e in env)) + + def rest(self, body: dict) -> None: + (self.state / "rest").write_text(json.dumps(body)) + + def file(self, relative: str, content: str) -> Path: + path = self.root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + def run(self, *, docker: bool = True) -> str: + command = service_clusters_command("docker" if docker else None) + assert command.count("/run/systemd/system") == 1 + command = command.replace("/run/systemd/system", str(self.booted)) + return subprocess.run( + ["/bin/sh", "-c", command], + capture_output=True, + text=True, + env={"PATH": str(self.bin)}, + timeout=30, + ).stdout + + +def _exec_start(*argv: str) -> str: + return f"ExecStart={{ path={argv[0]} ; argv[]={' '.join(argv)} ; ignore_errors=no ; pid=7 ; status=0/0 }}" + + +@pytest.fixture(params=_awks()) +def host(request, tmp_path) -> _Host: + return _Host(tmp_path, request.param) + + +class TestTheCommandOnAHost: + def test_patroni_as_on_db_01(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) + host.rest(DB01_CLUSTER) + + out = host.run() + + assert SECRET not in out + assert "10.7.234.13" not in out # the DCS endpoints stay on the host + assert "not-the-member-name" not in out + (cluster,) = parse_service_clusters(out)["patroni"] + assert cluster["cluster"] == "netork-db" + assert [(m["name"], m["role"], m["this_host"]) for m in cluster["members"]] == [ + ("db-01", "leader", True), + ("db-02", "sync_standby", False), + ] + + def test_patroni_whose_rest_api_does_not_answer(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) + + (cluster,) = parse_service_clusters(host.run())["patroni"] + + assert [(m["name"], m["address"], m["role"]) for m in cluster["members"]] == [ + ("db-01", "10.7.234.12", None) + ] + + @pytest.mark.skipif(os.geteuid() == 0, reason="root reads a file of mode 000") + def test_a_patroni_config_it_cannot_read_is_not_looked_at(self, host): + config = host.file("etc/patroni/config.yml", PATRONI_YML) + config.chmod(0) + host.unit( + "patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)) + ) + + out = host.run() + + assert SECRET not in out + assert parse_service_clusters(out)["patroni"] is None + + def test_the_sync_and_a_worker_without_their_secrets(self, host): + host.container( + "adguardhome-sync", + "ghcr.io/bakito/adguardhome-sync:v0.9.3", + "ORIGIN_URL=http://10.9.100.5:3000", + f"ORIGIN_PASSWORD={SECRET}", + f"REPLICA1_URL=http://admin:{SECRET}@10.9.100.6:3000", + f"REPLICA1_PASSWORD={SECRET}", + "CRON=*/5 * * * *", + ) + host.container( + "bunkerweb", + "bunkerity/bunkerweb@sha256:2026", + "API_WHITELIST_IP=10.50.100.5", + f"DATABASE_URI=mariadb+pymysql://bw:{SECRET}@db/bw", + ) + host.container("zabbix", "zabbix/zabbix-agent2:alpine-latest", f"ZBX_PSK={SECRET}") + + out = host.run() + + assert SECRET not in out + assert "admin" not in out + reading = parse_service_clusters(out) + assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == [ + "10.9.100.5", + "10.9.100.6", + ] + assert reading["bunkerweb"][0]["cluster"] == "10.50.100.5" + assert reading["patroni"] == [] + + def test_a_host_without_docker(self, host): + reading = parse_service_clusters(host.run(docker=False)) + + assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], []) + + +class TestTheMixin: + class _Driver(ServiceClustersMixin): + def __init__(self, answers: dict) -> None: + self.answers = answers + self.calls: list = [] + + def _run_service_clusters_command(self, command: str, *, privileged: bool) -> str: + self.calls.append(privileged) + return self.answers[privileged] + + def test_root_first(self): + driver = self._Driver({True: _wire(), False: "unused"}) + + assert driver.get_service_clusters()["patroni"] == [] + assert driver.calls == [True] + + def test_without_root_when_root_brought_no_report(self): + driver = self._Driver({True: "sudo: a password is required", False: _wire()}) + + assert driver.get_service_clusters()["bunkerweb"] == [] + assert driver.calls == [True, False] + + def test_the_container_engine_binary_is_the_drivers(self): + class _Qnap(self._Driver): + def _container_engine_binary(self, engine: str) -> str: + return "/share/CACHEDEV1_DATA/.qpkg/container-station/bin/docker" + + driver = _Qnap({True: _wire(), False: _wire()}) + seen: list = [] + + def run(command: str, *, privileged: bool) -> str: + seen.append(command) + return _wire() + + driver._run_service_clusters_command = run # type: ignore[method-assign] + driver.get_service_clusters() + + assert "container-station/bin/docker" in seen[0] + + def test_the_default_command_asks_docker(self): + assert "docker" in SERVICE_CLUSTERS_COMMAND + + +class TestTheSource: + """Where the evidence was read: the unit or container that carries it.""" + + def test_a_template_unit_is_named_after_its_instance(self): + reading = parse_service_clusters( + _wire(_patroni("active active", "scope pg", "name a", unit="patroni@16-main.service")) + ) + + assert reading["patroni"][0]["source"] == "patroni@16-main" + + def test_a_container_by_its_name(self): + reading = parse_service_clusters( + _wire( + "", + _container( + "adguardhome-sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000" + ), + _container("bw", "bunkerity/bunkerweb", "API_WHITELIST_IP 10.0.0.5"), + ) + ) + + assert reading["adguardhome_sync"][0]["source"] == "adguardhome-sync" + assert reading["bunkerweb"][0]["source"] == "bw" diff --git a/tests/test_service_packages.py b/tests/test_service_packages.py new file mode 100644 index 0000000..0a1a45b --- /dev/null +++ b/tests/test_service_packages.py @@ -0,0 +1,150 @@ +"""get_service_packages: which installed packages a systemd service is (NetOrk/netork#1236). + +A vulnerability is found on a *package*; a decision about a *service* -- the +postgresql unit, nginx -- means something only once it is known which packages +that service is. The package that ships its unit file and the one that ships the +binary it starts (``nginx-common`` and ``nginx-core``, ``postgresql-common`` and +``postgresql-16``) are what the unit is made of. Reading that is the same on +every systemd host -- ``systemctl show`` and ``dpkg -S`` or ``rpm -qf`` -- so the +command and its parse live here once, and a driver only carries the command. + +Only the unit's own packages: the libraries its binary links against are every +service's at once, and a decision about nginx is not one about libc. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess + +import pytest + +from napalm_device_types import ServicePackagesMixin +from napalm_device_types.service_packages import ( + SERVICE_PACKAGES_COMMAND, + parse_service_packages, +) + +#: ``systemctl show`` prints the properties in its own order: the name may follow its paths. +NGINX = """FragmentPath=/lib/systemd/system/nginx.service +ExecStart={ path=/usr/sbin/nginx ; argv[]=/usr/sbin/nginx -g daemon on; master_process on; ; ignore_errors=no ; start_time=[n/a] ; stop_time=[n/a] ; pid=0 ; code=(null) ; status=0/0 } +Id=nginx.service +""" +CRON = """Id=cron.service +FragmentPath=/usr/lib/systemd/system/cron.service +ExecStart={ path=/usr/sbin/cron ; argv[]=/usr/sbin/cron -f $EXTRA_OPTS ; ignore_errors=no } +""" + + +def _wire(units: str, owners: str = "", *, tool: str = "dpkg", noise: str = "") -> str: + return f"{noise}SVCPKG_BEGIN\n[units]\n{units}\n[{tool}]\n{owners}SVCPKG_END\n" + + +DPKG = """nginx-common: /lib/systemd/system/nginx.service +nginx-core: /usr/sbin/nginx +cron: /lib/systemd/system/cron.service +cron: /usr/sbin/cron +""" + + +class TestParsing: + def test_the_unit_file_and_the_binary_name_its_packages(self): + assert parse_service_packages(_wire(NGINX, DPKG)) == { + "nginx": ["nginx-common", "nginx-core"] + } + + def test_dpkg_is_asked_across_the_usr_merge(self): + """dpkg knows ``/lib/systemd/system/cron.service``; the unit says ``/usr/lib/...``.""" + assert parse_service_packages(_wire(CRON, DPKG)) == {"cron": ["cron"]} + + def test_several_units_are_kept_apart(self): + found = parse_service_packages(_wire(NGINX + "\n" + CRON, DPKG)) + + assert found == {"nginx": ["nginx-common", "nginx-core"], "cron": ["cron"]} + + def test_rpm_names_its_packages_too(self): + units = """Id=postgresql-16.service +FragmentPath=/usr/lib/systemd/system/postgresql-16.service +ExecStart={ path=/usr/pgsql-16/bin/postmaster ; argv[]=/usr/pgsql-16/bin/postmaster -D ${PGDATA} } +""" + owners = ( + "/usr/lib/systemd/system/postgresql-16.service\tpostgresql16-server \n" + "/lib/systemd/system/postgresql-16.service\tfile /lib/systemd/system/postgresql-16.service is not owned by any package\n" + "/usr/pgsql-16/bin/postmaster\tpostgresql16-server \n" + ) + + found = parse_service_packages(_wire(units, owners, tool="rpm")) + + assert found == {"postgresql-16": ["postgresql16-server"]} + + def test_an_architecture_and_several_owners_are_read(self): + units = "Id=foo.service\nExecStart={ path=/usr/bin/foo ; argv[]=/usr/bin/foo }\n" + owners = "libfoo1:amd64, foo-bin: /usr/bin/foo\n" + + assert parse_service_packages(_wire(units, owners)) == {"foo": ["foo-bin", "libfoo1"]} + + def test_a_unit_no_package_owns_is_read_as_owning_none(self): + """A unit file an admin wrote: read, and nothing owns it -- not unknown.""" + units = "Id=local-backup.service\nFragmentPath=/etc/systemd/system/local-backup.service\n" + + assert parse_service_packages(_wire(units)) == {"local-backup": []} + + def test_a_diversion_is_no_owner(self): + units = "Id=x.service\nExecStart={ path=/bin/sh ; argv[]=/bin/sh -c x }\n" + owners = "diversion by dash from: /bin/sh\ndash: /bin/sh\n" + + assert parse_service_packages(_wire(units, owners)) == {"x": ["dash"]} + + def test_whatever_surrounds_the_frame_is_ignored(self): + noise = "Last login: today\nprintf '%s%s\\n' SVCPKG_ BEGIN; ...\n" + + found = parse_service_packages(_wire(NGINX, DPKG, noise=noise)) + + assert found == {"nginx": ["nginx-common", "nginx-core"]} + + def test_a_report_cut_short_raises(self): + """A list cut short must never read as services that own nothing.""" + with pytest.raises(ValueError): + parse_service_packages("SVCPKG_BEGIN\n[units]\n" + NGINX) + + +class TestTheCommand: + def test_the_frame_is_not_in_the_command_itself(self): + assert "SVCPKG_BEGIN" not in SERVICE_PACKAGES_COMMAND + assert "SVCPKG_END" not in SERVICE_PACKAGES_COMMAND + + def test_it_writes_nothing(self): + for verb in ("sudo", " > ", ">>", " rm ", "install", "restart"): + assert verb not in SERVICE_PACKAGES_COMMAND + + @pytest.mark.skipif( + not os.path.isdir("/run/systemd/system") or shutil.which("dpkg") is None, + reason="reads a systemd host with dpkg", + ) + def test_it_runs_and_parses_on_this_host(self): + out = subprocess.run( + ["sh", "-c", SERVICE_PACKAGES_COMMAND], capture_output=True, text=True, timeout=60 + ).stdout + + found = parse_service_packages(out) + + assert found, "a systemd host has services" + assert any(found.values()), "and a package owns at least one of them" + assert all(isinstance(p, str) and p for pkgs in found.values() for p in pkgs) + + +class TestTheMixin: + def test_it_hands_its_command_to_the_driver_and_parses_the_answer(self): + class Driver(ServicePackagesMixin): + def __init__(self): + self.sent = [] + + def _run_service_packages_command(self, command: str) -> str: + self.sent.append(command) + return _wire(NGINX, DPKG) + + driver = Driver() + + assert driver.get_service_packages() == {"nginx": ["nginx-common", "nginx-core"]} + assert driver.sent == [SERVICE_PACKAGES_COMMAND] diff --git a/tests/test_snmp_health_profile.py b/tests/test_snmp_health_profile.py new file mode 100644 index 0000000..b4f2a10 --- /dev/null +++ b/tests/test_snmp_health_profile.py @@ -0,0 +1,71 @@ +"""SNMP health: drivers declare data, the application polls (docs/STRUCTURE.md). + +``SnmpHealthProfile`` is the shape a driver states its device's OIDs in. +``HealthMetricsMixin``, which collected in the base package, is deprecated. +""" + +from __future__ import annotations + +import asyncio +from typing import ClassVar, get_type_hints + +import pytest + +from napalm_device_types import ( + HealthMetricsMixin, + SnmpHealthProfile, + SnmpInterfacesSpec, + SnmpMemorySpec, + SnmpValueSpec, +) + + +def test_a_profile_says_where_each_value_lives(): + assert set(get_type_hints(SnmpHealthProfile)) == {"uptime", "cpu", "memory", "interfaces"} + assert set(get_type_hints(SnmpValueSpec)) == {"oids", "table", "fold", "pattern", "unknown"} + assert set(get_type_hints(SnmpMemorySpec)) == {"total", "used", "free", "percent", "unit"} + assert set(get_type_hints(SnmpInterfacesSpec)) == {"skip", "tx_errors_are_drops"} + + +def test_every_key_is_optional(): + for spec in (SnmpHealthProfile, SnmpValueSpec, SnmpMemorySpec, SnmpInterfacesSpec): + assert spec.__required_keys__ == frozenset(), spec.__name__ + + +def test_a_driver_declares_its_profile_as_a_class_attribute(): + class Driver: + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = { + "cpu": {"oids": ["1.3.6.1.4.1.9.1", "1.3.6.1.4.1.9.2"]}, + "memory": {"percent": {"table": "1.3.6.1.4.1.9.3", "fold": "first"}}, + } + + assert Driver.SNMP_HEALTH_PROFILE["cpu"]["oids"][0] == "1.3.6.1.4.1.9.1" + + +def test_a_value_the_agent_gets_wrong_is_declared_unknown(): + """Neither the standard reading nor a default: the application reports none.""" + assert get_type_hints(SnmpValueSpec)["unknown"] is bool + + class Driver: + SNMP_HEALTH_PROFILE: ClassVar[SnmpHealthProfile] = {"cpu": {"unknown": True}} + + assert Driver.SNMP_HEALTH_PROFILE["cpu"].get("unknown") is True + + +def test_the_shapes_live_in_their_own_module_and_models_still_exports_them(): + from napalm_device_types import models, snmp_health_profile + + for name in ("SnmpHealthProfile", "SnmpValueSpec", "SnmpMemorySpec", "SnmpInterfacesSpec"): + assert getattr(models, name) is getattr(snmp_health_profile, name), name + assert getattr(snmp_health_profile, name).__module__ == snmp_health_profile.__name__ + + +def test_the_collector_in_the_base_package_warns_that_it_is_deprecated(): + async def get(oid): + return None + + async def walk(oid): + return {} + + with pytest.warns(DeprecationWarning, match="SNMP_HEALTH_PROFILE"): + asyncio.run(HealthMetricsMixin.get_health_metrics(get, walk)) diff --git a/tests/test_ssh_host_keys.py b/tests/test_ssh_host_keys.py new file mode 100644 index 0000000..6470349 --- /dev/null +++ b/tests/test_ssh_host_keys.py @@ -0,0 +1,277 @@ +"""SSH host keys: trust on first use, and a changed key refused before any login. + +The layer above keeps the store (NetOrk/netork#1167). A driver gets the key it +should expect through ``optional_args["ssh_host_keys"]`` (by port) and reports every key it +sees through ``optional_args["ssh_host_key_observer"]``. These tests run real +paramiko against an in-process SSH server, because the property that matters — +nothing is sent to a host with the wrong key — is paramiko's behaviour once the +key is loaded, not something a fake could show. +""" + +from __future__ import annotations + +import os +import socket +import threading + +import paramiko +import pytest +from napalm.base.exceptions import ConnectionException + +from napalm_device_types.ssh_host_keys import ( + KNOWN_KEYS_ARG, + OBSERVER_ARG, + HostKeyCheck, + SshHostKey, + SshHostKeyMismatch, +) + +#: An ed25519 public key and the fingerprint ``ssh-keygen -lf`` prints for it. +ED25519 = SshHostKey( + "ssh-ed25519", "AAAAC3NzaC1lZDI1NTE5AAAAIJH2oO06vroJT2ooSB3/2tu8HufXQng4xQlxAk/4raee" +) +ED25519_FP = "SHA256:26dxfTuve20LGGTpJ1T0ns11/ID3XMhbOahQq3k75TU" + + +class _Server(paramiko.ServerInterface): + """Accepts any password and remembers each one it was offered.""" + + def __init__(self) -> None: + self.passwords: list[str] = [] + + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + self.passwords.append(password) + return paramiko.AUTH_SUCCESSFUL + + +class _Host: + """An SSH server on one end of a socket pair, with *host_key*.""" + + def __init__(self, host_key: paramiko.PKey) -> None: + self.client_sock, server_sock = socket.socketpair() + self.transport = paramiko.Transport(server_sock) + self.transport.add_server_key(host_key) + self.server = _Server() + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + try: + self.transport.start_server(server=self.server) + except Exception: # the client hung up on a refused key + pass + + def close(self) -> None: + self.transport.close() + self.client_sock.close() + self._thread.join(timeout=5) + + +@pytest.fixture +def host_key() -> paramiko.PKey: + return paramiko.ECDSAKey.generate() + + +@pytest.fixture +def host(host_key): + h = _Host(host_key) + yield h + h.close() + + +def _connect(check: HostKeyCheck, host: _Host) -> paramiko.SSHClient: + client = paramiko.SSHClient() + check.connect( + client, + username="admin", + password="s3cret", + sock=host.client_sock, + look_for_keys=False, + allow_agent=False, + timeout=5, + ) + return client + + +def _check(expected: SshHostKey | None, seen: list, port: int = 22) -> HostKeyCheck: + args: dict = {OBSERVER_ARG: seen.append} + if expected is not None: + args[KNOWN_KEYS_ARG] = {str(port): expected.as_dict()} + return HostKeyCheck(args, "10.0.0.5", port) + + +class TestSshHostKey: + def test_fingerprint_is_the_one_ssh_keygen_prints(self): + assert ED25519.fingerprint == ED25519_FP + + def test_of_reads_a_paramiko_key(self, host_key): + key = SshHostKey.of(host_key) + assert key.key_type == host_key.get_name() + assert key.key == host_key.get_base64() + + def test_as_dict_round_trips(self): + d = ED25519.as_dict() + assert d == {"key_type": "ssh-ed25519", "key": ED25519.key, "fingerprint": ED25519_FP} + assert SshHostKey.from_value(d) == ED25519 + + @pytest.mark.parametrize("value", [None, {}, ""]) + def test_no_value_is_no_key(self, value): + assert SshHostKey.from_value(value) is None + + @pytest.mark.parametrize( + "value", [{"key_type": "ssh-ed25519"}, {"key": "AAAA"}, "ssh-ed25519 AAAA"] + ) + def test_a_malformed_value_is_refused(self, value): + with pytest.raises(ValueError): + SshHostKey.from_value(value) + + def test_known_hosts_line_names_a_port_other_than_22(self): + assert ED25519.known_hosts_line("10.0.0.5", 22) == f"10.0.0.5 ssh-ed25519 {ED25519.key}" + line = ED25519.known_hosts_line("10.0.0.5", 2222) + assert line == f"[10.0.0.5]:2222 ssh-ed25519 {ED25519.key}" + + +class TestKnownKeys: + def test_the_key_is_the_one_for_the_port_connected_to(self): + args = {KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}} + assert HostKeyCheck(args, "10.0.0.5", 22).expected == ED25519 + assert HostKeyCheck(args, "10.0.0.5", 2222).expected is None + + def test_a_port_may_be_given_as_a_number(self): + args = {KNOWN_KEYS_ARG: {2222: ED25519.as_dict()}} + assert HostKeyCheck(args, "10.0.0.5", 2222).expected == ED25519 + + def test_a_malformed_mapping_is_refused(self): + with pytest.raises(ValueError): + HostKeyCheck({KNOWN_KEYS_ARG: [ED25519.as_dict()]}, "10.0.0.5", 22) + + +class TestParamiko: + def test_first_use_accepts_the_key_and_reports_it(self, host, host_key): + seen: list = [] + _connect(_check(None, seen), host).close() + assert host.server.passwords == ["s3cret"] + assert seen == [ + { + "host": "10.0.0.5", + "port": 22, + "seen": SshHostKey.of(host_key).as_dict(), + "expected": None, + "accepted": True, + } + ] + + def test_the_known_key_connects(self, host, host_key): + seen: list = [] + known = SshHostKey.of(host_key) + _connect(_check(known, seen), host).close() + assert host.server.passwords == ["s3cret"] + assert seen[0]["accepted"] is True + assert seen[0]["expected"] == known.as_dict() + + def test_a_changed_key_is_refused_before_any_login(self, host, host_key): + seen: list = [] + other = SshHostKey.of(paramiko.ECDSAKey.generate()) + with pytest.raises(SshHostKeyMismatch) as caught: + _connect(_check(other, seen), host) + assert host.server.passwords == [] + assert caught.value.observation.seen == SshHostKey.of(host_key) + assert caught.value.observation.expected == other + assert seen[0]["accepted"] is False + assert seen[0]["seen"]["fingerprint"] == SshHostKey.of(host_key).fingerprint + + def test_a_key_of_another_type_is_refused(self, host): + seen: list = [] + with pytest.raises(SshHostKeyMismatch): + _connect(_check(ED25519, seen), host) + assert host.server.passwords == [] + + def test_the_mismatch_is_a_connection_failure_that_names_both_keys(self, host, host_key): + with pytest.raises(ConnectionException) as caught: + _connect(_check(ED25519, []), host) + message = str(caught.value) + assert "10.0.0.5:22" in message + assert SshHostKey.of(host_key).fingerprint in message + assert ED25519_FP in message + + def test_a_failing_observer_does_not_break_the_connection(self, host): + def boom(_observation): + raise RuntimeError("store down") + + check = HostKeyCheck({OBSERVER_ARG: boom}, "10.0.0.5", 22) + _connect(check, host).close() + assert host.server.passwords == ["s3cret"] + + def test_without_optional_args_it_still_connects(self, host): + _connect(HostKeyCheck(None, "10.0.0.5", 22), host).close() + assert host.server.passwords == ["s3cret"] + + +class TestNetmiko: + BASE = { + "ssh_strict": False, + "alt_host_keys": True, + "alt_key_file": "/x", + "system_host_keys": True, + "conn_timeout": 9, + } + + def test_first_use_lets_netmiko_accept_the_key(self): + with HostKeyCheck({}, "10.0.0.5", 22).netmiko(self.BASE) as kwargs: + assert kwargs == { + "ssh_strict": False, + "alt_host_keys": False, + "system_host_keys": False, + "conn_timeout": 9, + } + + def test_a_known_key_is_the_only_one_netmiko_takes(self): + check = HostKeyCheck({KNOWN_KEYS_ARG: {"2222": ED25519.as_dict()}}, "10.0.0.5", 2222) + with check.netmiko(self.BASE) as kwargs: + path = kwargs["alt_key_file"] + assert kwargs["ssh_strict"] is True + assert kwargs["alt_host_keys"] is True + assert kwargs["system_host_keys"] is False + assert kwargs["conn_timeout"] == 9 + with open(path) as f: + assert f.read().strip() == ED25519.known_hosts_line("10.0.0.5", 2222) + assert not os.path.exists(path) + + def test_a_refusal_inside_netmiko_becomes_the_mismatch(self, host_key): + # netmiko re-raises paramiko's BadHostKeyException as its own exception, + # inside the except block, and a driver wraps that once more. + seen: list = [] + check = _check(ED25519, seen) + with pytest.raises(SshHostKeyMismatch) as caught: + with check.netmiko({}): + try: + try: + other = paramiko.ECDSAKey.generate() + raise paramiko.BadHostKeyException("10.0.0.5", host_key, other) + except paramiko.SSHException as exc: + raise RuntimeError(f"A paramiko SSHException occurred: {exc}") from exc + except RuntimeError as exc: + raise ConnectionException(str(exc)) from exc + assert caught.value.observation.seen == SshHostKey.of(host_key) + assert caught.value.observation.expected == ED25519 + assert seen[0]["accepted"] is False + + def test_any_other_failure_passes_unchanged(self): + with pytest.raises(ConnectionException, match="timed out"): + with HostKeyCheck({KNOWN_KEYS_ARG: {"22": ED25519.as_dict()}}, "10.0.0.5", 22).netmiko( + {} + ): + raise ConnectionException("timed out") + + def test_observe_reports_the_key_netmiko_saw(self, host_key): + class _Transport: + def get_remote_server_key(self): + return host_key + + seen: list = [] + observation = _check(None, seen).observe(_Transport()) + assert observation.accepted is True + assert seen[0]["seen"] == SshHostKey.of(host_key).as_dict() diff --git a/tests/test_ssh_penalties.py b/tests/test_ssh_penalties.py new file mode 100644 index 0000000..69d16dc --- /dev/null +++ b/tests/test_ssh_penalties.py @@ -0,0 +1,250 @@ +"""Exempting the managing client from sshd's per-source penalties (NetOrk/netork#1800). + +OpenSSH 9.8 penalises a source whose connections do not authenticate and then +drops every connection from it. A security scanner's probes earn the managing +client exactly that, and its next session -- a poll, an Ansible run -- is +turned away. ``exempt_from_ssh_penalties`` puts the address the host sees the +client at into ``PerSourcePenaltyExemptList``, where that directive exists. + +The real outputs this is held to were taken from Debian 13 (OpenSSH 10.0p2) and +Debian 12 (OpenSSH 9.2p1, which knows no penalties and rejects the directive). +""" + +from __future__ import annotations + +import re + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.ssh_server import ( + DROPIN_DIR, + MAIN_CONFIG, + PENALTY_DROPIN_PATH, + OpenSshServerMixin, + client_address, + is_exempt, + parse_penalty_exemptions, +) + +#: ``sshd -T | grep -i penalt`` on Debian 13, OpenSSH 10.0p2, as installed. +DEBIAN_13 = ( + "persourcepenaltyexemptlist none\n" + "persourcepenalties crash:90 authfail:5 noauth:1 grace-exceeded:10 refuseconnection:10" + " max:600 min:15 max-sources4:65536 max-sources6:65536 overflow:permissive" + " overflow6:permissive\n" +) + + +# ── reading ────────────────────────────────────────────────────────────── + + +def test_a_new_sshd_has_penalties_and_no_exemption(): + assert parse_penalty_exemptions(DEBIAN_13) == [] + + +def test_the_exemptions_sshd_uses_are_read(): + text = DEBIAN_13.replace("none", "10.40.100.10,192.0.2.0/24") + + assert parse_penalty_exemptions(text) == ["10.40.100.10", "192.0.2.0/24"] + + +def test_an_sshd_before_9_8_has_no_penalties(): + """It prints neither line -- and refuses the directive, so nothing may be written.""" + assert parse_penalty_exemptions("port 22\nmacs hmac-sha2-256\n") is None + + +@pytest.mark.parametrize( + ("value", "address"), + [ + ("10.40.100.10 41866 10.40.100.118 22", "10.40.100.10"), + ("fd00::10 41866 fd00::118 22", "fd00::10"), + ("", None), + ("not-an-address 1 2 3", None), + ("10.40.100.10 41866", None), + ], +) +def test_the_client_address_comes_from_ssh_connection(value, address): + assert client_address(value) == address + + +def test_an_address_inside_an_exempt_network_is_exempt(): + assert is_exempt("10.40.100.10", ["192.0.2.0/24", "10.40.100.0/24"]) + assert not is_exempt("10.40.100.10", ["10.40.100.11", "bogus"]) + + +# ── the mixin, against a fake host ─────────────────────────────────────── + + +class PenaltyHost(OpenSshServerMixin): + """An sshd that reads the main config and the drop-in directory, first value wins.""" + + def __init__( + self, + *, + main: str = f"Include {DROPIN_DIR}/*.conf\nPort 22\n", + penalties: bool = True, + client: str = "10.40.100.10 41866 10.40.100.118 22", + ) -> None: + self.files: dict[str, str] = {MAIN_CONFIG: main} + self.penalties = penalties + self.client = client + self.syntax_ok = True + self.reloads = 0 + self.ran: list[str] = [] + + def _config_lines(self) -> list[str]: + lines: list[str] = [] + for line in self.files[MAIN_CONFIG].splitlines(): + if re.match(r"\s*include\s", line, re.I): + for path in sorted(p for p in self.files if p.startswith(DROPIN_DIR + "/")): + lines += self.files[path].splitlines() + elif re.match(r"\s*match\s", line, re.I): + break + else: + lines.append(line) + return lines + + def exempt(self) -> str: + for line in self._config_lines(): + parts = line.split() + if len(parts) == 2 and parts[0].lower() == "persourcepenaltyexemptlist": + return parts[1] + return "none" + + def run_command(self, command, *, privileged=False, timeout=60, stdin=None): + self.ran.append(command) + if "SSH_CONNECTION" in command: + assert not privileged, "it is the login session's, not sudo's" + return CommandResult(self.client, "", 0) + if "mktemp" in command: + path = f"/tmp/netork-ssh.{len(self.files)}" + self.files[path] = (stdin or b"").decode() + return CommandResult(path, "", 0) + assert privileged, command + m = re.search(r"install -m \"\$m\" -- '?([^' ]+)'? '?([^' ]+)'?", command) + if m: + target = re.search(r"mv -f -- '?[^' ]+'? '?([^' ;]+)'?", command).group(1) + self.files[target] = self.files.pop(m.group(1)) + return CommandResult("", "", 0) + if "sshd -T" in command: + if not self.penalties: + return CommandResult("port 22\n", "", 0) + return CommandResult(DEBIAN_13.replace("none", self.exempt()), "", 0) + if "sshd -t" in command: + return CommandResult( + "", "" if self.syntax_ok else "bad line 1", 0 if self.syntax_ok else 255 + ) + m = re.match(r"cat -- '?([^' ]+)'?$", command) + if m: + path = m.group(1) + if path in self.files: + return CommandResult(self.files[path], "", 0) + return CommandResult("", "No such file", 1) + m = re.match(r"rm -f -- '?([^' ]+)'?$", command) + if m: + self.files.pop(m.group(1), None) + return CommandResult("", "", 0) + if "reload" in command or "HUP" in command: + self.reloads += 1 + return CommandResult("", "", 0) + if command.startswith("mkdir"): + return CommandResult("", "", 0) + raise AssertionError(f"unexpected command: {command}") + + +def test_the_client_address_is_exempted_in_a_dropin(): + host = PenaltyHost() + lines: list[str] = [] + + result = host.exempt_from_ssh_penalties(log=lines.append) + + assert result.applied and not result.rolled_back + assert result.address == "10.40.100.10" + assert result.path == PENALTY_DROPIN_PATH + assert host.exempt() == "10.40.100.10" + assert host.reloads == 1 + assert lines + + +def test_the_hosts_own_exemptions_stay(): + host = PenaltyHost( + main=f"Include {DROPIN_DIR}/*.conf\nPerSourcePenaltyExemptList 192.0.2.0/24\n" + ) + + result = host.exempt_from_ssh_penalties() + + assert result.applied + assert host.exempt() == "192.0.2.0/24,10.40.100.10" + + +def test_nothing_is_written_to_an_sshd_without_penalties(): + """OpenSSH before 9.8 does not start with the directive in its config.""" + host = PenaltyHost(penalties=False) + + result = host.exempt_from_ssh_penalties() + + assert not result.applied and not result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + assert host.reloads == 0 + assert "9.8" in result.message + + +def test_nothing_changes_when_the_address_is_already_exempt(): + host = PenaltyHost( + main=f"Include {DROPIN_DIR}/*.conf\nPerSourcePenaltyExemptList 10.40.100.0/24\n" + ) + + result = host.exempt_from_ssh_penalties() + + assert not result.applied + assert PENALTY_DROPIN_PATH not in host.files + assert "already" in result.message + + +def test_an_sshd_config_without_the_include_is_left_alone(): + host = PenaltyHost(main="Port 22\n") + + result = host.exempt_from_ssh_penalties() + + assert not result.applied + assert host.files[MAIN_CONFIG] == "Port 22\n" + assert DROPIN_DIR in result.message + + +def test_a_rejected_config_is_taken_back(): + host = PenaltyHost() + host.syntax_ok = False + + result = host.exempt_from_ssh_penalties() + + assert result.rolled_back and not result.applied + assert PENALTY_DROPIN_PATH not in host.files + + +def test_an_exemption_sshd_does_not_use_is_taken_back(): + """A drop-in read before ours sets the directive; ours is valid and ignored.""" + host = PenaltyHost() + host.files[f"{DROPIN_DIR}/00-aaa.conf"] = "PerSourcePenaltyExemptList 192.0.2.1\n" + + result = host.exempt_from_ssh_penalties() + + assert result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + + +def test_it_is_taken_back_when_a_new_session_cannot_get_in(): + host = PenaltyHost() + + result = host.exempt_from_ssh_penalties(verify=lambda: False) + + assert result.rolled_back + assert PENALTY_DROPIN_PATH not in host.files + assert host.reloads == 2 + + +def test_a_session_without_a_client_address_raises(): + host = PenaltyHost(client="") + + with pytest.raises(RuntimeError): + host.exempt_from_ssh_penalties() diff --git a/tests/test_ssh_refused.py b/tests/test_ssh_refused.py new file mode 100644 index 0000000..f736006 --- /dev/null +++ b/tests/test_ssh_refused.py @@ -0,0 +1,168 @@ +"""A connection sshd turns away before the SSH banner is tried again (NetOrk/netork#1800). + +OpenSSH 9.8+ drops every connection from a source it has just penalised +(``PerSourcePenalties``): it accepts the TCP connection and closes it at once, +and paramiko reports "Error reading SSH protocol banner". The penalty lasts +seconds to minutes, so a connection that waits a little gets in. +""" + +from __future__ import annotations + +import socket +import threading + +import paramiko +import pytest +from napalm.base.exceptions import ConnectionException + +from napalm_device_types import ssh_host_keys +from napalm_device_types.ssh_host_keys import ( + REFUSED_RETRY_WAITS, + HostKeyCheck, + refused_before_banner, + retry_refused, +) + + +def _banner_error() -> paramiko.SSHException: + try: + try: + raise EOFError() + except EOFError as eof: + raise paramiko.SSHException("Error reading SSH protocol banner") from eof + except paramiko.SSHException as exc: + return exc + + +class TestWhatCountsAsRefused: + def test_paramikos_banner_error(self): + assert refused_before_banner(_banner_error()) + + def test_wrapped_by_a_driver(self): + try: + try: + raise _banner_error() + except paramiko.SSHException as exc: + raise ConnectionException(f"cannot connect: {exc}") from exc + except ConnectionException as wrapped: + assert refused_before_banner(wrapped) + + @pytest.mark.parametrize( + "exc", + [ + paramiko.AuthenticationException("Authentication failed."), + TimeoutError("timed out"), + ConnectionRefusedError(111, "Connection refused"), + ValueError("anything else"), + ], + ) + def test_not_a_failed_login_a_silent_host_or_a_closed_port(self, exc): + """Those do not end by waiting: retrying them only delays the error.""" + assert not refused_before_banner(exc) + + +class TestRetry: + def test_it_waits_and_tries_again(self): + slept: list[float] = [] + attempts = iter([_banner_error(), _banner_error(), "connected"]) + + def attempt(): + value = next(attempts) + if isinstance(value, BaseException): + raise value + return value + + assert retry_refused(attempt, sleep=slept.append) == "connected" + assert slept == list(REFUSED_RETRY_WAITS[:2]) + + def test_it_gives_up_with_the_last_error(self): + slept: list[float] = [] + + def attempt(): + raise _banner_error() + + with pytest.raises(paramiko.SSHException): + retry_refused(attempt, sleep=slept.append) + assert slept == list(REFUSED_RETRY_WAITS) + + def test_other_errors_are_not_retried(self): + slept: list[float] = [] + + def attempt(): + raise paramiko.AuthenticationException("Authentication failed.") + + with pytest.raises(paramiko.AuthenticationException): + retry_refused(attempt, sleep=slept.append) + assert slept == [] + + def test_the_waits_outlast_the_penalty_seen(self): + """16 s on a new Ubuntu 26.04 VM on .50.""" + assert sum(REFUSED_RETRY_WAITS) > 16 + + +class _PenalisingServer: + """A TCP server that closes the first *drops* connections at once, as a penalised + source's are, and then speaks SSH with any password.""" + + def __init__(self, drops: int) -> None: + self.drops = drops + self.listener = socket.create_server(("127.0.0.1", 0)) + self.port = self.listener.getsockname()[1] + self.key = paramiko.ECDSAKey.generate() + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + while True: + try: + conn, _ = self.listener.accept() + except OSError: + return + if self.drops: + self.drops -= 1 + conn.close() + continue + transport = paramiko.Transport(conn) + transport.add_server_key(self.key) + + class _Any(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + return paramiko.AUTH_SUCCESSFUL + + try: + transport.start_server(server=_Any()) + except Exception: + pass + self.transport = transport + return + + def close(self) -> None: + self.listener.close() + transport = getattr(self, "transport", None) + if transport: + transport.close() + + +def test_host_key_check_connects_once_the_penalty_is_over(monkeypatch): + server = _PenalisingServer(drops=2) + slept: list[float] = [] + monkeypatch.setattr(ssh_host_keys.time, "sleep", slept.append) + client = paramiko.SSHClient() + try: + HostKeyCheck({}, "127.0.0.1", server.port).connect( + client, + username="admin", + password="s3cret", + look_for_keys=False, + allow_agent=False, + timeout=5, + banner_timeout=5, + ) + assert client.get_transport().is_authenticated() + assert slept == list(REFUSED_RETRY_WAITS[:2]) + finally: + client.close() + server.close() diff --git a/tests/test_ssh_server.py b/tests/test_ssh_server.py index 462cea9..ad1db25 100644 --- a/tests/test_ssh_server.py +++ b/tests/test_ssh_server.py @@ -9,7 +9,6 @@ cannot get through -- runs without a machine. from __future__ import annotations import re -from typing import Dict, List, Optional import pytest @@ -69,7 +68,11 @@ def test_parse_reads_the_four_lists_in_server_order(): eff = parse_sshd_effective(SSHD_T) assert eff["kex"][:2] == ["sntrup761x25519-sha512@openssh.com", "curve25519-sha256"] assert eff["hostkey"] == ["ssh-ed25519", "ecdsa-sha2-nistp256", "rsa-sha2-512", "rsa-sha2-256"] - assert eff["cipher"] == ["chacha20-poly1305@openssh.com", "aes128-ctr", "aes256-gcm@openssh.com"] + assert eff["cipher"] == [ + "chacha20-poly1305@openssh.com", + "aes128-ctr", + "aes256-gcm@openssh.com", + ] assert eff["mac"][-1] == "hmac-sha2-256" @@ -82,7 +85,9 @@ def test_parse_refuses_output_without_the_lists(): def test_plan_removes_what_was_asked_and_keeps_the_order(): - plan = plan_ssh_hardening(parse_sshd_effective(SSHD_T), {"mac": ["hmac-sha1", "umac-64-etm@openssh.com"]}) + plan = plan_ssh_hardening( + parse_sshd_effective(SSHD_T), {"mac": ["hmac-sha1", "umac-64-etm@openssh.com"]} + ) assert plan.remove == {"mac": ["umac-64-etm@openssh.com", "hmac-sha1"]} assert plan.keep["mac"] == ["hmac-sha2-256-etm@openssh.com", "hmac-sha2-256"] assert plan.changed_classes == ["mac"] @@ -113,20 +118,38 @@ def test_plan_keeps_one_algorithm_the_managing_client_still_speaks(): know would leave the server reachable, but not by netOrk.""" plan = plan_ssh_hardening( parse_sshd_effective(SSHD_T), - {"kex": ["curve25519-sha256@libssh.org", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", - "diffie-hellman-group14-sha256"]}, + { + "kex": [ + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] + }, client=PARAMIKO, ) assert "curve25519-sha256@libssh.org" in plan.keep["kex"] - assert [(r.algorithm, r.reason) for r in plan.refused] == [("curve25519-sha256@libssh.org", "client")] - assert plan.remove["kex"] == ["ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "diffie-hellman-group14-sha256"] + assert [(r.algorithm, r.reason) for r in plan.refused] == [ + ("curve25519-sha256@libssh.org", "client") + ] + assert plan.remove["kex"] == [ + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] def test_plan_without_a_client_list_only_guards_the_last_algorithm(): plan = plan_ssh_hardening( parse_sshd_effective(SSHD_T), - {"kex": ["curve25519-sha256@libssh.org", "ecdh-sha2-nistp256", "ecdh-sha2-nistp384", - "diffie-hellman-group14-sha256"]}, + { + "kex": [ + "curve25519-sha256@libssh.org", + "ecdh-sha2-nistp256", + "ecdh-sha2-nistp384", + "diffie-hellman-group14-sha256", + ] + }, ) assert plan.refused == [] assert plan.keep["kex"] == ["sntrup761x25519-sha512@openssh.com", "curve25519-sha256"] @@ -171,12 +194,18 @@ def test_plan_removes_a_host_key_algorithm_when_a_usable_one_stays(): def test_plan_leaves_host_keys_alone_when_the_keys_are_unknown(): plan = plan_ssh_hardening(parse_sshd_effective(SSHD_T), {"hostkey": ["ecdsa-sha2-nistp256"]}) assert plan.changed_classes == [] - assert [(r.algorithm, r.reason) for r in plan.refused] == [("ecdsa-sha2-nistp256", "host_keys_unknown")] + assert [(r.algorithm, r.reason) for r in plan.refused] == [ + ("ecdsa-sha2-nistp256", "host_keys_unknown") + ] def test_host_key_algorithms_from_ssh_keygen(): assert host_key_algorithms("\n".join(KEYGEN.values())) == { - "ecdsa-sha2-nistp256", "ssh-ed25519", "rsa-sha2-512", "rsa-sha2-256", "ssh-rsa", + "ecdsa-sha2-nistp256", + "ssh-ed25519", + "rsa-sha2-512", + "rsa-sha2-256", + "ssh-rsa", } assert host_key_algorithms("521 SHA256:x no comment (ECDSA)") == {"ecdsa-sha2-nistp521"} assert host_key_algorithms("") is None @@ -204,11 +233,15 @@ def test_include_is_found_only_in_the_global_section(): assert has_dropin_include("Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n") assert has_dropin_include(" include /etc/ssh/sshd_config.d/*.conf\n") assert not has_dropin_include("Port 22\n") - assert not has_dropin_include("Port 22\nMatch User x\n Include /etc/ssh/sshd_config.d/*.conf\n") + assert not has_dropin_include( + "Port 22\nMatch User x\n Include /etc/ssh/sshd_config.d/*.conf\n" + ) def test_main_config_gets_the_lists_first_and_the_old_lines_disabled(): - main = "# defaults\nPort 22\nMACs hmac-sha1,hmac-sha2-256\nMatch User backup\n MACs hmac-sha1\n" + main = ( + "# defaults\nPort 22\nMACs hmac-sha1,hmac-sha2-256\nMatch User backup\n MACs hmac-sha1\n" + ) text = render_main_config(main, {"mac": ["hmac-sha2-256"]}, ["mac"]) lines = text.splitlines() first_directive = next(line for line in lines if line and not line.startswith("#")) @@ -227,28 +260,34 @@ def test_main_config_rewrite_is_idempotent(): # ── the mixin, against a fake host ─────────────────────────────────────── -_DIRECTIVE = {"kexalgorithms": "kex", "hostkeyalgorithms": "hostkey", "ciphers": "cipher", "macs": "mac"} +_DIRECTIVE = { + "kexalgorithms": "kex", + "hostkeyalgorithms": "hostkey", + "ciphers": "cipher", + "macs": "mac", +} class FakeHost(OpenSshServerMixin): """A host with files, an sshd that reads them first-match-wins, and a log of what ran.""" - def __init__(self, main: str = "Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n", - sshd_t: str = SSHD_T) -> None: - self.files: Dict[str, str] = {MAIN_CONFIG: main} - self.modes: Dict[str, str] = {MAIN_CONFIG: "644"} + def __init__( + self, main: str = "Include /etc/ssh/sshd_config.d/*.conf\nPort 22\n", sshd_t: str = SSHD_T + ) -> None: + self.files: dict[str, str] = {MAIN_CONFIG: main} + self.modes: dict[str, str] = {MAIN_CONFIG: "644"} self.sshd_t = sshd_t self.defaults = parse_sshd_effective(sshd_t) self.keys = {p: KEYGEN[p] for p in parse_sshd_hostkeys(sshd_t)} self.keygen_ok = True - self.ran: List[str] = [] + self.ran: list[str] = [] self.syntax_ok = True self.reloads = 0 - def _effective(self) -> Dict[str, List[str]]: + def _effective(self) -> dict[str, list[str]]: eff = {k: list(v) for k, v in self.defaults.items()} seen = set() - lines: List[str] = [] + lines: list[str] = [] for line in self.files[MAIN_CONFIG].splitlines(): if re.match(r"\s*include\s", line, re.I): lines += self.files.get(DROPIN_PATH, "").splitlines() @@ -269,8 +308,14 @@ class FakeHost(OpenSshServerMixin): #: or a cached timestamp): the password the driver put in front of the data. SUDO_PASSWORD = "hunter2" - def run_command(self, command: str, *, privileged: bool = False, timeout: float = 60, - stdin: Optional[bytes] = None) -> CommandResult: + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: self.ran.append(command) if privileged and stdin is not None: stdin = f"{self.SUDO_PASSWORD}\n".encode() + stdin @@ -294,14 +339,20 @@ class FakeHost(OpenSshServerMixin): if not self.keygen_ok: return CommandResult("", "Permission denied", 1) paths = re.search(r"for k in (.*?); do", command).group(1).replace("'", "").split() - return CommandResult("".join(self.keys[p] + "\n" for p in paths if p in self.keys), "", 0) + return CommandResult( + "".join(self.keys[p] + "\n" for p in paths if p in self.keys), "", 0 + ) if command.startswith("sshd -T") or "sshd -T" in command: eff = self._effective() out = "".join(f"{d} {','.join(eff[k])}\n" for d, k in _DIRECTIVE.items()) - out += "".join(line + "\n" for line in self.sshd_t.splitlines() if line.startswith("hostkey ")) + out += "".join( + line + "\n" for line in self.sshd_t.splitlines() if line.startswith("hostkey ") + ) return CommandResult(out, "", 0) if "sshd -t" in command: - return CommandResult("", "" if self.syntax_ok else "bad line 3", 0 if self.syntax_ok else 255) + return CommandResult( + "", "" if self.syntax_ok else "bad line 3", 0 if self.syntax_ok else 255 + ) m = re.match(r"cat -- '?([^' ]+)'?$", command) if m: path = m.group(1) @@ -323,7 +374,7 @@ class FakeHost(OpenSshServerMixin): def test_mixin_writes_the_dropin_checks_it_and_reloads(): host = FakeHost() - lines: List[str] = [] + lines: list[str] = [] result = host.harden_ssh_server({"mac": ["hmac-sha1"]}, client=PARAMIKO, log=lines.append) assert result.applied and not result.rolled_back assert result.path == DROPIN_PATH @@ -439,7 +490,7 @@ def test_mixin_reports_a_failed_way_back_instead_of_raising(): def test_mixin_refuses_to_lock_out_a_host_with_only_ed25519_and_rsa_keys(): host = FakeHost(sshd_t=SSHD_T_FEW_KEYS) - lines: List[str] = [] + lines: list[str] = [] result = host.harden_ssh_server( {"hostkey": ["ssh-ed25519", "rsa-sha2-256", "rsa-sha2-512"]}, log=lines.append ) diff --git a/tests/test_sudo_history.py b/tests/test_sudo_history.py new file mode 100644 index 0000000..76d62e6 --- /dev/null +++ b/tests/test_sudo_history.py @@ -0,0 +1,395 @@ +"""The sudo password an earlier driver left in shell histories, and taking it back. + +The scripts are run for real, with ``sh``, against a home directory in a temp +directory: what they count, delete and report is what a host would see. Root's +part -- giving files back to root -- runs with a ``chown`` that only records, +so the test needs no root. +""" + +from __future__ import annotations + +import getpass +import os +import subprocess +from pathlib import Path + +import pytest + +from napalm_device_types.channel import CommandResult +from napalm_device_types.sudo_history import HISTORY_BACKUP_SUFFIX, SudoHistoryMixin + +MARKER = "| sudo -S -p " + +LEAKED = "echo 'hunter2' | sudo -S -p \"\" apt-get update\n" +OTHER = "ls -la\n" + + +class _Host(SudoHistoryMixin): + """A driver whose host is this machine, with *home* as the login user's home.""" + + SUDO_HISTORY_MARKER = MARKER + + def __init__(self, home: Path, bin_dir: Path, *, root: bool = False) -> None: + self.home = home + self.bin_dir = bin_dir + self.root = root + #: Who ``id -un`` says the login user is; None for this machine's user name. + self.user: str | None = None + self.ran: list[tuple[str, bool]] = [] + + def run_command( + self, + command: str, + *, + privileged: bool = False, + timeout: float = 60, + stdin: bytes | None = None, + ) -> CommandResult: + # The password goes ahead of a privileged command's stdin: there must be none. + assert not (privileged and stdin is not None) + self.ran.append((command, privileged)) + env = { + **os.environ, + "HOME": str(self.home), + "PATH": f"{self.bin_dir}:{os.environ['PATH']}", + } + if command.strip() == "id -u; id -un" and self.root: + return CommandResult("0\nroot\n", "", 0) + if command.strip() == "id -u; id -un": + # Never this machine's uid: CI runs as root, which the owner step skips. + return CommandResult(f"4242\n{self.user or getpass.getuser()}\n", "", 0) + done = subprocess.run( # noqa: S603 - the test's own scripts + ["sh", "-c", command], capture_output=True, text=True, env=env, timeout=timeout + ) + return CommandResult(done.stdout, done.stderr, done.returncode) + + +@pytest.fixture +def bin_dir(tmp_path: Path) -> Path: + """A ``chown`` that records its arguments instead of changing anything.""" + d = tmp_path / "bin" + d.mkdir() + chown = d / "chown" + chown.write_text(f'#!/bin/sh\nprintf "%s\\n" "$*" >> {tmp_path / "chowned"}\n') + chown.chmod(0o755) + return d + + +@pytest.fixture +def home(tmp_path: Path) -> Path: + h = tmp_path / "home" + h.mkdir() + return h + + +def _chowned(tmp_path: Path) -> list[str]: + log = tmp_path / "chowned" + return log.read_text().splitlines() if log.exists() else [] + + +class TestCounting: + def test_counts_the_lines_per_history_file_without_reading_them_out(self, home, bin_dir): + (home / ".bash_history").write_text(OTHER + LEAKED * 3) + (home / ".zsh_history").write_text(": 1700000000:0;" + LEAKED) + (home / ".sh_history").write_text(OTHER) + host = _Host(home, bin_dir) + + found = host.get_history_sudo_lines() + + assert found == [ + {"file": "~/.bash_history", "lines": 3}, + {"file": "~/.zsh_history", "lines": 1}, + ] + # Counted as the login user, and nothing of the content came back. + assert all(not privileged for _cmd, privileged in host.ran) + assert "hunter2" not in repr(found) + + def test_no_history_no_lines(self, home, bin_dir): + assert _Host(home, bin_dir).get_history_sudo_lines() == [] + + def test_the_pattern_is_a_fixed_string(self, home, bin_dir): + # "-S" is no regex trouble, but "|" would be an alternation in an ERE. + (home / ".bash_history").write_text("sudo -S -p\n| sudo\n") + assert _Host(home, bin_dir).get_history_sudo_lines() == [] + + def test_a_driver_that_never_typed_the_password_counts_nothing(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + host.SUDO_HISTORY_MARKER = None + + assert host.get_history_sudo_lines() == [] + assert host.ran == [] + + def test_an_unreadable_history_raises_rather_than_reporting_none(self, home, bin_dir): + if os.geteuid() == 0: + pytest.skip("root reads every file") + history = home / ".bash_history" + history.write_text(LEAKED) + history.chmod(0) + try: + with pytest.raises(RuntimeError): + _Host(home, bin_dir).get_history_sudo_lines() + finally: + history.chmod(0o600) + + +class TestRemoving: + def test_deletes_exactly_those_lines_and_says_how_many(self, home, bin_dir): + (home / ".bash_history").write_text(OTHER + LEAKED + "cd /etc\n" + LEAKED) + (home / ".ash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + + result = host.remove_history_sudo_lines() + + assert result["success"] is True + assert (home / ".bash_history").read_text() == OTHER + "cd /etc\n" + assert (home / ".ash_history").read_text() == "" + assert result["removed"] == 3 + assert result["files"] == [ + {"file": "~/.bash_history", "before": 2, "after": 0}, + {"file": "~/.ash_history", "before": 1, "after": 0}, + ] + assert "~/.bash_history: 2 lines removed (2 before, 0 after)" in result["output"] + + def test_no_backup_with_the_password_is_left_behind(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + + _Host(home, bin_dir).remove_history_sudo_lines() + + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + assert not (home / f".bash_history{HISTORY_BACKUP_SUFFIX}").exists() + + def test_the_history_runs_as_the_login_user(self, home, bin_dir): + (home / ".bash_history").write_text(LEAKED) + host = _Host(home, bin_dir) + + host.remove_history_sudo_lines() + + history_scripts = [(c, p) for c, p in host.ran if ".bash_history" in c] + assert history_scripts and all(not p for _c, p in history_scripts) + + def test_a_clean_history_is_left_as_it_is(self, home, bin_dir): + history = home / ".bash_history" + history.write_text(OTHER) + before = history.stat().st_ino + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is True + assert result["removed"] == 0 + assert result["files"] == [{"file": "~/.bash_history", "before": 0, "after": 0}] + assert history.stat().st_ino == before # not rewritten + + def test_a_history_that_is_a_link_is_left_alone_and_the_fix_fails( + self, home, bin_dir, tmp_path + ): + target = tmp_path / "elsewhere" + target.write_text(LEAKED) + (home / ".bash_history").symlink_to(target) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert target.read_text() == LEAKED + assert (home / ".bash_history").is_symlink() + assert "~/.bash_history: a symbolic link, left alone (1 line)" in result["output"] + + def test_the_lines_of_a_driver_that_never_typed_the_password_are_not_looked_for( + self, home, bin_dir + ): + host = _Host(home, bin_dir) + host.SUDO_HISTORY_MARKER = None + + result = host.remove_history_sudo_lines() + + assert result["success"] is True + assert result["removed"] == 0 + + def test_an_unreadable_history_fails_the_step(self, home, bin_dir): + if os.geteuid() == 0: + pytest.skip("root reads every file") + history = home / ".bash_history" + history.write_text(LEAKED) + history.chmod(0) + try: + result = _Host(home, bin_dir).remove_history_sudo_lines() + finally: + history.chmod(0o600) + + assert result["success"] is False + assert "~/.bash_history: cannot be read" in result["output"] + + def test_a_sed_that_fails_keeps_the_lines_and_leaves_no_backup(self, home, bin_dir): + (bin_dir / "sed").write_text( + '#!/bin/sh\nfor a; do f="$a"; done\ncp "$f" "$f.napalm-bak"\nexit 4\n' + ) + (bin_dir / "sed").chmod(0o755) + (home / ".bash_history").write_text(LEAKED * 2) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert "~/.bash_history: could not be rewritten (2 lines kept)" in result["output"] + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + + def test_a_sed_that_lost_the_file_gets_it_back(self, home, bin_dir): + (bin_dir / "sed").write_text( + '#!/bin/sh\nfor a; do f="$a"; done\nmv "$f" "$f.napalm-bak"\nexit 4\n' + ) + (bin_dir / "sed").chmod(0o755) + (home / ".bash_history").write_text(OTHER + LEAKED) + + result = _Host(home, bin_dir).remove_history_sudo_lines() + + assert result["success"] is False + assert (home / ".bash_history").read_text() == OTHER + LEAKED + assert sorted(p.name for p in home.iterdir()) == [".bash_history"] + + def test_a_script_that_dies_says_why(self, home, bin_dir): + host = _Host(home, bin_dir) + host.run_command = lambda command, **kw: CommandResult("", "sh: killed", 137) # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "sh: killed" in result["output"] + + def test_a_line_it_does_not_know_is_ignored(self): + files: list = [] + said: list[str] = [] + + assert SudoHistoryMixin._history_line(["what", "ever"], files, said) is True + assert SudoHistoryMixin._history_line([], files, said) is True + assert files == [] and said == [] + + +class TestOwners: + def _host(self, home, bin_dir, files): + host = _Host(home, bin_dir) + host.SUDO_ROOT_FILES = tuple(str(f) for f in files) + return host + + def test_a_file_the_login_user_owns_goes_back_to_root(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert _chowned(tmp_path) == [f"-h 0:0 {conf}"] + assert result["owners_restored"] == [str(conf)] + assert result["success"] is True + + def test_only_as_root_and_without_stdin(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + + host.remove_history_sudo_lines() + + privileged = [c for c, p in host.ran if p] + assert len(privileged) == 1 and "chown" in privileged[0] + # The login user's name is quoted into root's script, never taken from stdin. + assert getpass.getuser() in privileged[0] + + def test_a_link_or_a_missing_file_is_not_touched(self, home, bin_dir, tmp_path): + real = tmp_path / "real" + real.write_text("x") + link = tmp_path / "00proxy" + link.symlink_to(real) + + result = self._host(home, bin_dir, [link, tmp_path / "missing"]).remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["owners_restored"] == [] + assert result["success"] is True + + def test_a_file_someone_else_owns_is_not_touched(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + host.user = "nobody-here-4242" + + result = host.remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["owners_restored"] == [] + + def test_a_root_login_has_nothing_to_give_back(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + host.root = True + + result = host.remove_history_sudo_lines() + + assert _chowned(tmp_path) == [] + assert result["success"] is True + + def test_the_owners_are_set_even_when_the_history_could_not_be_cleaned( + self, home, bin_dir, tmp_path + ): + target = tmp_path / "elsewhere" + target.write_text(LEAKED) + (home / ".bash_history").symlink_to(target) + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert result["success"] is False + assert [s["step"] for s in result["steps"]] == ["history", "owners"] + assert result["owners_restored"] == [str(conf)] + + def test_without_knowing_the_login_user_nothing_changes_owner(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + real = host.run_command + + def run(command, **kw): + if command.startswith("id "): + return CommandResult("", "id: not found", 127) + return real(command, **kw) + + host.run_command = run # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "cannot tell who the login user is" in result["output"] + assert _chowned(tmp_path) == [] + + def test_a_sudo_that_refuses_fails_the_step(self, home, bin_dir, tmp_path): + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + host = self._host(home, bin_dir, [conf]) + real = host.run_command + + def run(command, *, privileged=False, **kw): + if privileged: + return CommandResult("", "sudo: a password is required", 1) + return real(command, privileged=privileged, **kw) + + host.run_command = run # type: ignore[method-assign] + + result = host.remove_history_sudo_lines() + + assert result["success"] is False + assert "sudo: a password is required" in result["output"] + + def test_a_refused_chown_fails_the_step(self, home, bin_dir, tmp_path): + (bin_dir / "chown").write_text("#!/bin/sh\necho 'Operation not permitted' >&2\nexit 1\n") + conf = tmp_path / "snmpd.conf" + conf.write_text("x") + + result = self._host(home, bin_dir, [conf]).remove_history_sudo_lines() + + assert result["success"] is False + assert result["owners_restored"] == [] + assert f"{conf}: could not be given back to root" in result["output"] + + +def test_exported_from_the_package(): + import napalm_device_types + + assert napalm_device_types.SudoHistoryMixin is SudoHistoryMixin diff --git a/tests/test_system_identity.py b/tests/test_system_identity.py index 2dc8d8b..c874b00 100644 --- a/tests/test_system_identity.py +++ b/tests/test_system_identity.py @@ -38,8 +38,20 @@ def test_a_supermicro_placeholder_serial_is_dropped(): @pytest.mark.parametrize( "value", - ["", "None", "N/A", "Not Specified", "To be filled by O.E.M.", "Default string", - "System Serial Number", "0123456789", "1234567890", "0", "00000000", "Not Applicable"], + [ + "", + "None", + "N/A", + "Not Specified", + "To be filled by O.E.M.", + "Default string", + "System Serial Number", + "0123456789", + "1234567890", + "0", + "00000000", + "Not Applicable", + ], ) def test_placeholders(value): assert is_placeholder_identity(value) @@ -47,9 +59,12 @@ def test_placeholders(value): @pytest.mark.parametrize( "uuid", - ["00000000-0000-0000-0000-000000000000", "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", - # A UUID many cheap boards ship, the same on every one of them. - "03000200-0400-0500-0006-000700080009"], + [ + "00000000-0000-0000-0000-000000000000", + "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF", + # A UUID many cheap boards ship, the same on every one of them. + "03000200-0400-0500-0006-000700080009", + ], ) def test_placeholder_uuids(uuid): assert parse_system_identity(_out(uuid, "", ""))["system_uuid"] == "" @@ -63,7 +78,10 @@ def test_real_values_are_not_placeholders(): def test_unreadable_files_read_as_empty(): # product_serial and product_uuid are root's alone; without root they come back empty. assert parse_system_identity(_out("", "", "BS1")) == { - "system_uuid": "", "product_serial": "", "board_serial": "BS1"} + "system_uuid": "", + "product_serial": "", + "board_serial": "BS1", + } def test_output_without_its_frame_is_not_a_reading(): diff --git a/tests/test_systemd_services.py b/tests/test_systemd_services.py index 9423baf..7d9695b 100644 --- a/tests/test_systemd_services.py +++ b/tests/test_systemd_services.py @@ -409,4 +409,4 @@ class TestSystemdServicesMixin: def test_not_every_os_driver_has_it(self): assert not hasattr(OSDriver, "manage_service") - assert callable(getattr(SystemdServicesMixin, "manage_service")) + assert callable(SystemdServicesMixin.manage_service) diff --git a/tests/test_tls_certificates.py b/tests/test_tls_certificates.py new file mode 100644 index 0000000..ab0f817 --- /dev/null +++ b/tests/test_tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +The layer above keeps the store (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +A driver gets the fingerprint it should expect through ``optional_args["tls_fingerprints"]`` +(by port) and reports every certificate it sees through +``optional_args["tls_certificate_observer"]``. The handshake tests run real TLS against an +in-process server on a socket pair: what matters is the certificate the server presents in +a real handshake, which a fake could not show. +""" + +from __future__ import annotations + +import datetime +import hashlib +import socket +import ssl +import threading + +import pytest +import requests +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID +from napalm.base.exceptions import ConnectionException + +from napalm_device_types.tls_certificates import ( + KNOWN_FINGERPRINTS_ARG, + OBSERVER_ARG, + CertificateCheck, + TlsCertificateMismatch, + fetch_certificate, + fingerprint, + normalise_fingerprint, +) + +DER = b"not really a certificate, but bytes are bytes" +DER_HEX = hashlib.sha256(DER).hexdigest().upper() +DER_FP = "SHA256:" + ":".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)) +OTHER_FP = "SHA256:" + ":".join(["AB"] * 32) + + +def _self_signed(tmp_path, name: str = "switch") -> tuple[bytes, str, str]: + """A self-signed certificate, as a ProCurve ships one: (DER, cert file, key file).""" + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + now = datetime.datetime.now(datetime.UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .sign(key, hashes.SHA256()) + ) + cert_file, key_file = tmp_path / f"{name}.crt", tmp_path / f"{name}.key" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + return cert.public_bytes(serialization.Encoding.DER), str(cert_file), str(key_file) + + +class _TlsServer: + """A TLS server on one end of a socket pair; it handshakes and then waits.""" + + def __init__(self, cert_file: str, key_file: str) -> None: + self.client_sock, server_sock = socket.socketpair() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_file, key_file) + self._server_sock = server_sock + self._context = context + self.received = b"" + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + try: + tls = self._context.wrap_socket(self._server_sock, server_side=True) + self.received = tls.recv(1024) + except (OSError, ssl.SSLError): # the client hung up after the handshake + pass + + def connect(self, address, timeout=None): + return self.client_sock + + def close(self) -> None: + self.client_sock.close() + self._server_sock.close() + self._thread.join(timeout=5) + + +@pytest.fixture +def certificate(tmp_path): + return _self_signed(tmp_path) + + +@pytest.fixture +def server(certificate): + _der, cert_file, key_file = certificate + s = _TlsServer(cert_file, key_file) + yield s + s.close() + + +def _check(expected: str | None, seen: list, port: int = 443) -> CertificateCheck: + args: dict = {OBSERVER_ARG: seen.append} + if expected is not None: + args[KNOWN_FINGERPRINTS_ARG] = {str(port): expected} + return CertificateCheck(args, "10.0.0.5", port) + + +class TestFingerprint: + def test_is_sha256_of_the_der_certificate_in_colon_hex(self): + assert fingerprint(DER) == DER_FP + + @pytest.mark.parametrize( + "value", + [ + DER_FP, + DER_FP.lower(), + DER_HEX, + DER_HEX.lower(), + DER_FP.removeprefix("SHA256:"), + " ".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)), + "sha256:" + DER_HEX, + ], + ) + def test_a_pin_may_be_written_as_openssl_or_a_browser_shows_it(self, value): + assert normalise_fingerprint(value) == DER_FP + + @pytest.mark.parametrize( + "value", ["", "AB:CD", "SHA1:" + DER_HEX, "ZZ" * 32, DER_HEX + "AB", 42] + ) + def test_anything_else_is_refused(self, value): + with pytest.raises(ValueError): + normalise_fingerprint(value) + + +class TestKnownFingerprints: + def test_the_pin_is_the_one_for_the_port_connected_to(self): + args = {KNOWN_FINGERPRINTS_ARG: {"443": DER_HEX}} + assert CertificateCheck(args, "10.0.0.5", 443).expected == DER_FP + assert CertificateCheck(args, "10.0.0.5", 8443).expected is None + + def test_a_port_may_be_given_as_a_number(self): + args = {KNOWN_FINGERPRINTS_ARG: {8443: DER_FP}} + assert CertificateCheck(args, "10.0.0.5", 8443).expected == DER_FP + + def test_a_malformed_mapping_is_refused(self): + with pytest.raises(ValueError): + CertificateCheck({KNOWN_FINGERPRINTS_ARG: [DER_FP]}, "10.0.0.5", 443) + + def test_without_optional_args_it_is_first_use(self): + assert CertificateCheck(None, "10.0.0.5", 443).expected is None + + +class TestVerify: + def test_first_use_accepts_and_reports_only_once_observed(self): + seen: list = [] + check = _check(None, seen) + observation = check.verify(DER) + assert observation.accepted is True + assert seen == [] + check.observe() + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": None, + "accepted": True, + } + ] + + def test_the_pinned_certificate_is_accepted(self): + seen: list = [] + check = _check(DER_HEX.lower(), seen) + assert check.verify(DER).accepted is True + check.observe() + assert seen[0]["expected"] == {"fingerprint": DER_FP} + assert seen[0]["accepted"] is True + + def test_another_certificate_is_refused_reported_and_named(self): + seen: list = [] + with pytest.raises(TlsCertificateMismatch) as caught: + _check(OTHER_FP, seen).verify(DER) + assert caught.value.observation.seen == DER_FP + assert caught.value.observation.expected == OTHER_FP + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": {"fingerprint": OTHER_FP}, + "accepted": False, + } + ] + + def test_the_mismatch_is_a_connection_failure_that_names_both(self): + with pytest.raises(ConnectionException) as caught: + _check(OTHER_FP, []).verify(DER) + message = str(caught.value) + assert "10.0.0.5:443" in message + assert DER_FP in message + assert OTHER_FP in message + + def test_a_failing_observer_does_not_break_the_connection(self): + def boom(_observation): + raise RuntimeError("store down") + + check = CertificateCheck({OBSERVER_ARG: boom}, "10.0.0.5", 443) + check.verify(DER) + check.observe() + + def test_observe_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).observe() + + +class TestHandshake: + def test_fetch_returns_the_certificate_the_server_presents(self, server, certificate): + der, _cert, _key = certificate + assert fetch_certificate("10.0.0.5", 443, 5, connect=server.connect) == der + + def test_verify_server_refuses_another_certificate_before_anything_is_sent( + self, server, tmp_path + ): + other_der, _c, _k = _self_signed(tmp_path, "impostor") + seen: list = [] + check = _check(fingerprint(other_der), seen) + with pytest.raises(TlsCertificateMismatch): + check.verify_server(5, connect=server.connect) + server.close() + assert server.received == b"" + assert seen[0]["accepted"] is False + + def test_verify_server_accepts_on_first_use(self, server, certificate): + der, _cert, _key = certificate + observation = _check(None, []).verify_server(5, connect=server.connect) + assert observation.seen == fingerprint(der) + + +class TestRequests: + def test_pin_makes_every_connection_present_the_verified_certificate(self): + check = _check(None, []) + check.verify(DER) + session = requests.Session() + check.pin(session) + adapter = session.get_adapter("https://10.0.0.5/rest/v7/") + assert adapter.poolmanager.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + proxy = adapter.proxy_manager_for("http://proxy.example:3128") + assert proxy.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + + def test_pin_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).pin(requests.Session()) + + def test_a_changed_certificate_on_a_pinned_session_is_a_mismatch(self): + from urllib3.util.ssl_ import assert_fingerprint + + seen: list = [] + check = _check(DER_FP, seen) + check.verify(DER) + impostor = b"another certificate" + with pytest.raises(TlsCertificateMismatch) as caught, check.refusing(): + try: + assert_fingerprint(impostor, DER_HEX) + except Exception as exc: + raise requests.exceptions.SSLError(exc) from exc + assert caught.value.observation.seen == fingerprint(impostor) + assert caught.value.observation.expected == DER_FP + assert seen[-1]["accepted"] is False + + def test_refusing_lets_other_errors_through(self): + check = _check(None, []) + check.verify(DER) + with pytest.raises(requests.exceptions.ConnectionError), check.refusing(): + raise requests.exceptions.ConnectionError("no route to host") diff --git a/tests/test_type_label.py b/tests/test_type_label.py index c949a5e..c215788 100644 --- a/tests/test_type_label.py +++ b/tests/test_type_label.py @@ -1,6 +1,7 @@ """Tests for TYPE_LABEL class attribute on all base driver classes.""" import pytest + from napalm_device_types import ( AccessPointDriver, FirewallDriver, @@ -12,7 +13,6 @@ from napalm_device_types import ( SwitchDriver, ) - BASE_CLASSES = [ (AccessPointDriver, "Access Point"), (FirewallDriver, "Firewall"), diff --git a/tests/test_ucd_metrics.py b/tests/test_ucd_metrics.py index eec7614..14fcadf 100644 --- a/tests/test_ucd_metrics.py +++ b/tests/test_ucd_metrics.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -from typing import Dict, Optional from napalm_device_types._ucd_metrics import ( OID_IF_DESCR, @@ -24,11 +23,11 @@ LEGACY = { } -def _collect(tables: Dict[str, Dict[str, str]]) -> dict: - async def get(oid: str) -> Optional[str]: +def _collect(tables: dict[str, dict[str, str]]) -> dict: + async def get(oid: str) -> str | None: return None - async def walk(oid: str) -> Dict[str, str]: + async def walk(oid: str) -> dict[str, str]: return tables.get(oid, {}) return asyncio.run(collect_ucd_metrics(get, walk)) @@ -44,7 +43,11 @@ def test_the_64_bit_counters_and_high_speed_win(): eth0 = _collect(tables)["interfaces"]["eth0"] - assert (eth0["rx_bytes"], eth0["tx_bytes"], eth0["speed_mbps"]) == (8589934592, 17179869184, 10000) + assert (eth0["rx_bytes"], eth0["tx_bytes"], eth0["speed_mbps"]) == ( + 8589934592, + 17179869184, + 10000, + ) def test_an_agent_without_ifxtable_falls_back_to_the_32_bit_counters(): diff --git a/tests/test_virtual_ips.py b/tests/test_virtual_ips.py index 8ec3cac..da54d55 100644 --- a/tests/test_virtual_ips.py +++ b/tests/test_virtual_ips.py @@ -152,9 +152,7 @@ class TestVipManager: assert parse_virtual_ips(_wire(section))["vip_manager"] == [] def test_a_template_unit_is_named_after_its_instance(self): - section = _vip_manager( - "active active", "cfg ip 10.0.0.10", unit="vip-manager@pg16.service" - ) + section = _vip_manager("active active", "cfg ip 10.0.0.10", unit="vip-manager@pg16.service") [entry] = parse_virtual_ips(_wire(section))["vip_manager"] @@ -496,8 +494,10 @@ class TestTheCommandOnAHost: assert SECRET not in out assert "MASTER" not in out and "priority" not in out reading = parse_virtual_ips(out) - assert [(e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) - for e in reading["keepalived"]] == [ + assert [ + (e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) + for e in reading["keepalived"] + ] == [ ("VI_WEB", "192.0.2.10", 24, "eth0", "vrid:51"), ("VI_DB", "192.0.2.20", 32, "eth1", "vrid:52"), ("VI_DB", "2001:db8::20", 64, "eth1", "vrid:52"), @@ -593,8 +593,10 @@ class TestTheCommandOnAHost: out = host.run() assert SECRET not in out - assert [(e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) - for e in parse_virtual_ips(out)["keepalived"]] == [ + assert [ + (e["instance"], e["address"], e["prefix_length"], e["interface"], e["group_key"]) + for e in parse_virtual_ips(out)["keepalived"] + ] == [ ("VI_1", "192.0.2.50", 32, "eth0", None), ("VI_2", "192.0.2.60", 24, "eth1", "vrid:53"), ] diff --git a/tests/test_vm_cpu_types.py b/tests/test_vm_cpu_types.py index 10a2474..61b8516 100644 --- a/tests/test_vm_cpu_types.py +++ b/tests/test_vm_cpu_types.py @@ -15,7 +15,7 @@ from __future__ import annotations import ast import inspect -from typing import List, get_type_hints +from typing import get_type_hints import napalm_device_types.hypervisor as hypervisor_module from napalm_device_types import HypervisorDriver @@ -33,7 +33,10 @@ def _declared(name: str) -> ast.FunctionDef: class TestCreateVmTakesACpuType: def test_cpu_type_is_an_optional_keyword(self): fn = _declared("create_vm_from_cloud_init") - kwonly = {arg.arg: default for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults)} + kwonly = { + arg.arg: default + for arg, default in zip(fn.args.kwonlyargs, fn.args.kw_defaults, strict=True) + } assert "cpu_type" in kwonly default = kwonly["cpu_type"] assert isinstance(default, ast.Constant) and default.value is None @@ -52,7 +55,7 @@ class TestCpuTypeListing: assert get_type_hints(VMCpuTypeDict) == { "name": str, "description": str, - "features": List[str], + "features": list[str], "available": bool, "default": bool, } diff --git a/tests/test_vm_identifiers.py b/tests/test_vm_identifiers.py index d38f6e2..3c7752c 100644 --- a/tests/test_vm_identifiers.py +++ b/tests/test_vm_identifiers.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import get_type_hints import pytest + from napalm_device_types.models import ( VMConfigDict, VMDict, diff --git a/tests/test_vm_migration.py b/tests/test_vm_migration.py new file mode 100644 index 0000000..18d7a8a --- /dev/null +++ b/tests/test_vm_migration.py @@ -0,0 +1,65 @@ +"""Moving a guest to another node of its cluster, and a node's HA maintenance mode. + +``get_vm_migration_preflight`` says where a guest could go and what holds it +back, ``migrate_vm`` moves it and waits, ``set_node_maintenance`` switches a +node's HA maintenance mode. Declared only, under ``TYPE_CHECKING``: netOrk +probes each with ``hasattr`` and hides what a driver lacks (VMware has none of +them yet). +""" + +from __future__ import annotations + +import ast +import inspect +from typing import get_type_hints + +import pytest + +import napalm_device_types.hypervisor as hypervisor_module +from napalm_device_types import HypervisorDriver, MigrationPreflightDict + + +def _declared(name: str) -> ast.FunctionDef: + tree = ast.parse(inspect.getsource(hypervisor_module)) + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == name: + return node + raise AssertionError(f"HypervisorDriver does not declare {name}()") + + +def _signature(name: str) -> str: + fn = _declared(name) + return f"({ast.unparse(fn.args)}) -> {ast.unparse(fn.returns) if fn.returns else None}" + + +class TestDeclarations: + @pytest.mark.parametrize( + "method", ["get_vm_migration_preflight", "migrate_vm", "set_node_maintenance"] + ) + def test_absent_until_a_driver_implements_it(self, method): + assert not hasattr(HypervisorDriver, method) + + def test_preflight(self): + assert _signature("get_vm_migration_preflight") == ( + "(self, name: str) -> MigrationPreflightDict" + ) + + def test_migrate(self): + assert _signature("migrate_vm") == ( + "(self, name: str, target_node: str, online: bool=True, " + "target_storage: str | None=None, timeout: int=3600) -> None" + ) + + def test_maintenance(self): + assert _signature("set_node_maintenance") == "(self, node: str, enabled: bool) -> None" + + +class TestPreflightShape: + def test_keys(self): + assert get_type_hints(MigrationPreflightDict) == { + "running": bool, + "allowed_nodes": list[str], + "refused_nodes": dict[str, str], + "local_disks": list[str], + "local_resources": list[str], + } diff --git a/tests/test_vm_provision_incomplete.py b/tests/test_vm_provision_incomplete.py new file mode 100644 index 0000000..ddaef8d --- /dev/null +++ b/tests/test_vm_provision_incomplete.py @@ -0,0 +1,53 @@ +"""A provisioning that failed after the hypervisor gave the VM its id says +which VM it left behind, so the caller can remove it (NAPALM/napalm-proxmox#28).""" + +from __future__ import annotations + +import pickle + +import pytest + +from napalm_device_types import VMProvisionIncomplete +from napalm_device_types.provisioning import VMProvisionIncomplete as from_module + + +def _raise_from(cause: Exception) -> VMProvisionIncomplete: + try: + try: + raise cause + except Exception as exc: + raise VMProvisionIncomplete(str(exc), vmid="101", node="pve1") from exc + except VMProvisionIncomplete as caught: + return caught + + +def test_it_is_exported_from_the_package_root(): + assert VMProvisionIncomplete is from_module + + +def test_it_names_the_vm_left_behind(): + caught = _raise_from(RuntimeError("disk import failed")) + assert (caught.vmid, caught.node) == ("101", "pve1") + + +def test_it_keeps_the_failure_and_its_words(): + cause = RuntimeError("disk import failed") + caught = _raise_from(cause) + assert caught.__cause__ is cause + assert str(caught) == "disk import failed" + + +def test_a_caller_catching_runtime_error_still_catches_it(): + # The contract has create_vm_from_cloud_init raise RuntimeError on failure. + with pytest.raises(RuntimeError): + raise VMProvisionIncomplete("start failed", vmid="7", node="n") + + +def test_vmid_and_node_are_keyword_only(): + with pytest.raises(TypeError): + VMProvisionIncomplete("failed", "101", "pve1") # type: ignore[misc] + + +def test_it_survives_pickling_as_a_task_result_would(): + caught = pickle.loads(pickle.dumps(VMProvisionIncomplete("failed", vmid="101", node="pve1"))) + assert (str(caught), caught.vmid, caught.node) == ("failed", "101", "pve1") diff --git a/tests/test_vm_stop_request.py b/tests/test_vm_stop_request.py new file mode 100644 index 0000000..144a3fc --- /dev/null +++ b/tests/test_vm_stop_request.py @@ -0,0 +1,28 @@ +"""begin_stop_vm: ask for a VM's clean shutdown and return at once. + +``stop_vm`` blocks until the VM is stopped, with a limit of the driver's own. +A caller that keeps its waits as timers (netOrk's graph workflows, +NetOrk/netork#1636) asks with ``begin_stop_vm`` and reads ``get_vm_status`` +until the VM is stopped. A driver mixes it in when it can; no role demands it. +""" + +from __future__ import annotations + +import inspect + +from napalm_device_types import HypervisorDriver, VmStopRequestMixin + + +def test_no_role_demands_it(): + assert not issubclass(HypervisorDriver, VmStopRequestMixin) + + +def test_declared_not_implemented(): + assert not hasattr(VmStopRequestMixin, "begin_stop_vm") + + +def test_declaration_documents_the_contract(): + source = inspect.getsource(VmStopRequestMixin) + assert "def begin_stop_vm(self, name: str) -> None" in source + assert "ValueError" in source + assert "RuntimeError" in source