From 5daba5d37cacae7404c57e2c7722272492ac2644 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 5 Nov 2026 00:24:24 +0100 Subject: [PATCH 1/2] feat: which logs a host writes, and who writes each (4.30.0) LogSourcesMixin.get_log_sources() reads, in one sh -c call as root, the units journald holds entries for, the log files running processes hold open (/proc//fd: *.log, *.log.N, anything under /var/log) with size and last write from stat -L on the descriptor, and the engine's own file of each container's output as that container's stream. Each file names the service or container of its first writer from /proc//cgroup; a file a container writes keeps the container's own path. journald's store, deleted files and a database's write-ahead log (LevelDB's 000004.log) are left out. Without root it reads again and says so (attributed: false); a host whose find cannot list open files (busybox) raises LogSourcesUnavailable rather than reading as no files. Refs NetOrk/netork#2049 --- CHANGELOG.md | 11 ++ README.md | 12 ++ napalm_device_types/__init__.py | 11 ++ napalm_device_types/log_sources.py | 236 ++++++++++++++++++++++++++++ napalm_device_types/models.py | 31 ++++ pyproject.toml | 2 +- tests/test_log_sources.py | 243 +++++++++++++++++++++++++++++ 7 files changed, 545 insertions(+), 1 deletion(-) create mode 100644 napalm_device_types/log_sources.py create mode 100644 tests/test_log_sources.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 72af09f..92a9064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ recorded here; their history is the git log. ## [Unreleased] +## [4.30.0] - 2026-11-04 + +### Added +- `LogSourcesMixin.get_log_sources()`: every log a Linux host writes. Sources are the units + journald holds entries for, the log files running processes hold open (with size, last write + and the service or container writing them), and the engine's file of each container's output. + A file a container writes is named by the container's own path. `LOG_SOURCES_COMMAND` and + `parse_log_sources` are public, and a host whose `find` cannot list open files (busybox) + raises `LogSourcesUnavailable`. A driver supplies `_run_log_sources_command(command, + privileged=)`, as for listening sockets (NetOrk/netork#2049). + ## [4.29.0] - 2026-11-04 ### Added diff --git a/README.md b/README.md index b21b930..9bb4993 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,18 @@ without `ss` is read with `netstat -lntup` (OpenWrt's busybox, old net-tools); o cgroup names the procd service (`/services//`). A host with neither raises `ListeningSocketsUnavailable`. +`LogSourcesMixin` (`get_log_sources`) is mixed in the same way: every log the host writes, in +one round trip. The units journald holds entries for (`journalctl -F _SYSTEMD_UNIT`, services +only), the log files running processes hold open (`/proc//fd`: `*.log`, `*.log.N` and +anything under `/var/log`) with size and last write from `stat -L` on the descriptor, and the +engine's own file of a container's output (`/var/lib/docker/containers//…`) as that +container's stream. Each file names the service or container of its first writer from +`/proc//cgroup`; a file a container writes carries the container's own path. journald's +store, deleted files and a database's write-ahead log (LevelDB's `000004.log`) are left out; a +host whose `find` cannot list open files (busybox) raises `LogSourcesUnavailable`. A +driver supplies `_run_log_sources_command(command, privileged=)`; without root only the login +user's processes show their files, and the reading says so (`attributed: false`). + LLDP neighbours are rows: `LldpNeighborRow` is one neighbour as its local port heard it (chassis and port id with their type, name, description, management addresses, capabilities in NAPALM's names), and `lldp_neighbors(rows)` / `lldp_neighbors_detail(rows, diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 1f3c912..6ec5dea 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -50,6 +50,7 @@ instead of being restated on every role that happens to need it: * :class:`~napalm_device_types.kernel.KernelFactsMixin` * :class:`~napalm_device_types.listening.ListeningSocketsMixin` * :class:`~napalm_device_types.local_accounts.LocalAccountsMixin` +* :class:`~napalm_device_types.log_sources.LogSourcesMixin` * :class:`~napalm_device_types.host_firewall.HostFirewallMixin` * :class:`~napalm_device_types.mac_acl.MacAclMixin` * :class:`~napalm_device_types.nat_vpn.NatVpnMixin` @@ -166,6 +167,12 @@ from napalm_device_types.local_accounts import ( LocalAccountDict, LocalAccountsMixin, ) +from napalm_device_types.log_sources import ( + LOG_SOURCES_COMMAND, + LogSourcesMixin, + LogSourcesUnavailable, + parse_log_sources, +) from napalm_device_types.mac_acl import MacAclMixin from napalm_device_types.management_controller import ManagementControllerDriver from napalm_device_types.media import MediaDriver @@ -398,6 +405,10 @@ __all__ = [ "parse_host_firewall", "ListeningSocketsMixin", "ListeningSocketsUnavailable", + "LOG_SOURCES_COMMAND", + "LogSourcesMixin", + "LogSourcesUnavailable", + "parse_log_sources", "parse_listening_sockets", "LLDPCTL_KEYVALUE_COMMAND", "LldpNeighborDict", diff --git a/napalm_device_types/log_sources.py b/napalm_device_types/log_sources.py new file mode 100644 index 0000000..99a60cf --- /dev/null +++ b/napalm_device_types/log_sources.py @@ -0,0 +1,236 @@ +"""Log sources: which logs a host writes, and who writes each of them. + +Before anything can watch a host's logs -- a person, CrowdSec, a syslog +forwarder -- it has to know which logs there are. That is read the same way on +every Linux host, so the command and its parse live here once, and a driver +only carries the command across (NetOrk/netork#2049). + +**Three kinds.** journald names every unit it holds entries for +(``journalctl -F _SYSTEMD_UNIT``); only services count, not login sessions. +``/proc//fd`` shows the files running processes hold open, a log being a +``*.log`` / ``*.log.N`` file or anything under ``/var/log``; ``stat -L`` on the +descriptor gives its size and last write, also for a file inside a container, +whose path is the container's own. The engine's own file of a container's +output (``/var/lib/docker/containers//…``) is that container's stream, not +a file of ``dockerd``. + +**Who writes it.** ``/proc//cgroup`` says which systemd service or +container the first process holding a file runs in, as for listening sockets +(:mod:`napalm_device_types.listening`). journald's own files, files already +deleted and a database's write-ahead log (LevelDB's ``000004.log``) are left out. + +**One round trip, root, and without it.** As for listening sockets the script +goes as one ``sh -c`` argument. Without root only the login user's processes +show their files; the command then runs again unprivileged, and the reading +says it is not ``attributed``. A report whose end is missing raises: a list cut +short must never read as logs that went away, and so must a host whose +``find`` cannot list open files (busybox): it raises +:class:`LogSourcesUnavailable`. +""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from shlex import quote +from typing import TYPE_CHECKING + +from napalm_device_types.listening import _cgroup_paths, _container, _sections, _unit +from napalm_device_types.models import LogSourceDict, LogSourcesDict +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "LOGS_BEGIN" +_END = "LOGS_END" + +#: One line, POSIX ``sh`` with GNU ``find`` (``-lname``, ``-printf``), read-only. +#: The frame markers are printed in two halves so that a transport which echoes +#: the command does not show them early. ``-printf '%p %l'`` puts the descriptor +#: before its target, so a target with a space in it stays whole. +LOG_SOURCES_COMMAND = ( + "PATH=$PATH:/usr/sbin:/sbin; " + "printf '%s%s\\n' LOGS_ BEGIN; " + "if [ -d /run/systemd/system ] && command -v journalctl >/dev/null 2>&1; then " + "j=journalctl; command -v timeout >/dev/null 2>&1 && j='timeout 30 journalctl'; " + "echo '[journald]'; $j -F _SYSTEMD_UNIT 2>/dev/null; fi; " + "if find / -maxdepth 0 -lname x -printf '' >/dev/null 2>&1; then " + "o=$(find /proc/[0-9]*/fd -mindepth 1 -maxdepth 1 -type l " + "\\( -lname '*.log' -o -lname '*.log.[0-9]*' -o -lname '/var/log/*' \\) " + "-printf '%p %l\\n' 2>/dev/null); " + "echo '[open]'; printf '%s\\n' \"$o\"; " + "echo '[stat]'; printf '%s\\n' \"$o\" | cut -d' ' -f1 " + "| xargs -r stat -L -c '%n %s %Y' 2>/dev/null; " + "pids=$(printf '%s\\n' \"$o\" | cut -d/ -f3 | sort -un); " + "echo '[cgroups]'; for p in $pids; do sed \"s|^|$p |\" /proc/$p/cgroup 2>/dev/null; done; " + "echo '[comm]'; for p in $pids; do c=$(cat /proc/$p/comm 2>/dev/null) && echo \"$p $c\"; done; " + "else echo '[no-find]'; fi; " + "printf '%s%s\\n' LOGS_ END" +) + +#: ``/proc/900/fd/5`` -- the process and its descriptor. +_FD_RE = re.compile(r"^/proc/(\d+)/fd/\d+$") +#: The engine's file of a container's output: json-file and local drivers. +_STREAM_RE = re.compile(r"^/var/lib/docker/containers/([0-9a-f]{64})/") +#: Files that are no log to read: journald's store, a file already deleted, and +#: the write-ahead log LevelDB and RocksDB name ``000004.log``. +_NOT_A_SOURCE = re.compile(r"^/var/log/journal/|\.journal~?$| \(deleted\)$|/\d{6}\.log$") +_SERVICE_SUFFIX = ".service" +_NO_FIND = "no-find" + + +class LogSourcesUnavailable(NotImplementedError): + """The host's ``find`` cannot list open files (busybox); nothing to read, nothing to retry.""" + + +def _frame(output: str) -> list[str]: + lines = [line.strip() for line in strip_terminal_codes(output).splitlines()] + try: + start = lines.index(_BEGIN) + end = lines.index(_END, start) + except ValueError: + raise ValueError("no intact log source report in the output") from None + return [line for line in lines[start + 1 : end] if line] + + +def _stats(lines: list[str]) -> dict[str, tuple[int, str]]: + """Each descriptor's size and last write: ``/proc/900/fd/5 2048 1762250200``.""" + stats: dict[str, tuple[int, str]] = {} + for line in lines: + fd, _, rest = line.partition(" ") + size, _, mtime = rest.partition(" ") + if size.isdigit() and mtime.isdigit(): + stats[fd] = (int(size), datetime.fromtimestamp(int(mtime), UTC).isoformat()) + return stats + + +def _names(lines: list[str]) -> dict[int, str]: + names: dict[int, str] = {} + for line in lines: + pid, _, name = line.partition(" ") + if pid.isdigit() and name: + names[int(pid)] = name + return names + + +def _journals(units: list[str]) -> list[LogSourceDict]: + services = {u[: -len(_SERVICE_SUFFIX)] for u in units if u.endswith(_SERVICE_SUFFIX)} + return [ + { + "kind": "journald", + "path": None, + "unit": unit, + "container_id": None, + "process": None, + "size": None, + "modified": None, + } + for unit in sorted(services) + ] + + +def _held(lines: list[str]) -> list[tuple[int, str, str]]: + """(pid, descriptor, target) for every log a process holds, by pid.""" + held = [] + for line in lines: + fd, _, target = line.partition(" ") + match = _FD_RE.match(fd) + if match and target and not _NOT_A_SOURCE.search(target): + held.append((int(match.group(1)), fd, target)) + return sorted(held) + + +def _files(sections: dict[str, list[str]]) -> list[LogSourceDict]: + stats = _stats(sections.get("stat", [])) + paths = _cgroup_paths(sections.get("cgroups", [])) + names = _names(sections.get("comm", [])) + found: dict[tuple[str, str | None, str], LogSourceDict] = {} + for pid, fd, target in _held(sections.get("open", [])): + stream = _STREAM_RE.match(target) + cgroup = paths.get(pid) + kind, container = ("container", stream.group(1)) if stream else ("file", _container(cgroup)) + source = found.get((kind, container, target)) + if source is None: + source = found[(kind, container, target)] = { + "kind": kind, + "path": target, + "unit": None if stream else _unit(cgroup), + "container_id": container, + "process": names.get(pid), + "size": None, + "modified": None, + } + if source["size"] is None and fd in stats: + source["size"], source["modified"] = stats[fd] + return list(found.values()) + + +def _order(source: LogSourceDict) -> tuple[str, str, str]: + return (source["kind"], source["container_id"] or "", source["path"] or source["unit"] or "") + + +def parse_log_sources(output: str) -> tuple[bool, list[LogSourceDict]]: + """Parse what :data:`LOG_SOURCES_COMMAND` printed: whether the host has a + journal, and every source, sorted by kind, container and name. + + :raises LogSourcesUnavailable: when the host has no GNU ``find``. + :raises ValueError: when the output carries no intact report. + """ + sections = _sections(_frame(output)) + if _NO_FIND in sections: + raise LogSourcesUnavailable("the host's find cannot list open files (no GNU find)") + journald = "journald" in sections + sources = _journals(sections.get("journald", [])) + _files(sections) + return journald, sorted(sources, key=_order) + + +class LogSourcesMixin: + """Adds :meth:`get_log_sources` to a driver that can run a command on a Linux host. + + The template form (README, "Function classes"): the command and its parse + are the same everywhere, so they are concrete here, and a driver supplies + only :meth:`_run_log_sources_command` -- how a command reaches its host, and + how it gains root there. Mixed in by the drivers that can, so + ``hasattr(driver, "get_log_sources")`` stays truthful. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_log_sources_command(self, command: str, *, privileged: bool) -> str: + """Run *command* on the host and return what it printed; as root + when *privileged*. The command is a single ``sh -c`` invocation.""" + ... + + def get_log_sources(self) -> LogSourcesDict: + """ + Returns every log the host writes, with the service or container writing it. + + * attributed (bool) - read as root, so every process's files are listed + * journald (bool) - the host keeps systemd's journal + * sources (list) - see :class:`~napalm_device_types.models.LogSourceDict` + + Example:: + + { + "attributed": True, + "journald": True, + "sources": [ + {"kind": "journald", "path": None, "unit": "nginx", + "container_id": None, "process": None, "size": None, "modified": None}, + {"kind": "file", "path": "/var/log/nginx/access.log", "unit": "nginx", + "container_id": None, "process": "nginx", "size": 2048, + "modified": "2026-11-04T09:56:40+00:00"}, + ], + } + + :raises LogSourcesUnavailable: if the host has no GNU ``find``. + :raises ValueError: if neither reading carried an intact report. + """ + command = f"sh -c {quote(LOG_SOURCES_COMMAND)}" + try: + output = self._run_log_sources_command(command, privileged=True) + journald, sources = parse_log_sources(output) + return {"attributed": True, "journald": journald, "sources": sources} + except ValueError: + pass + output = self._run_log_sources_command(command, privileged=False) + journald, sources = parse_log_sources(output) + return {"attributed": False, "journald": journald, "sources": sources} diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index b2d4302..32b27e2 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -406,6 +406,37 @@ class ListeningSocketsDict(TypedDict): sockets: list[ListeningSocketDict] +class LogSourceDict(TypedDict): + """A log the host writes: a unit's journal, a file a process holds open, or a + container's stream as the engine keeps it. + + ``path`` is as its writer sees it: a file a container writes is named inside + that container. Whether anything reads the log is the consumer's question. + """ + + kind: str # "journald", "file" or "container" + path: str | None # the file; None for a journal + unit: str | None # the journal's unit, or the file writer's service; without ".service" + container_id: str | None # the full container ID: the writer's, or the stream's + process: str | None # the first process holding the file + size: int | None # bytes; None for a journal, or when the file could not be read + modified: str | None # the last write, ISO 8601 UTC; None as for size + + +class LogSourcesDict(TypedDict): + """What ``LogSourcesMixin.get_log_sources`` read. + + ``attributed`` is False when the reading ran without root: only the login + user's own processes show their files then, so a missing log means "not + told", not "not there". ``journald`` is False on a host without systemd's + journal, where no unit has one. + """ + + attributed: bool + journald: bool + sources: list[LogSourceDict] + + class NetfilterMatchDict(TypedDict): """One condition of a netfilter rule. diff --git a/pyproject.toml b/pyproject.toml index b7a2a15..9c499b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "napalm-device-types" -version = "4.29.0" +version = "4.30.0" description = "Abstract device-type base classes for NAPALM drivers" readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_log_sources.py b/tests/test_log_sources.py new file mode 100644 index 0000000..5e767e1 --- /dev/null +++ b/tests/test_log_sources.py @@ -0,0 +1,243 @@ +"""get_log_sources: which logs a host writes, and who writes each of them. + +Before anything can watch a host's logs -- a person in a log window, CrowdSec, +a syslog forwarder -- something has to say which logs there are. Reading that +is the same on every Linux host: journald names the units it holds entries for, +``/proc//fd`` shows the log files running processes have open, and +``/proc//cgroup`` says which systemd service or container a writer runs in. +So all of it lives here once, and a driver only carries the command across +(NetOrk/netork#2049). +""" + +from __future__ import annotations + +import shutil +import subprocess + +import pytest + +from napalm_device_types.log_sources import ( + LOG_SOURCES_COMMAND, + LogSourcesMixin, + LogSourcesUnavailable, + parse_log_sources, +) + +CID = "4f1c" + "0" * 60 +OTHER = "9a2b" + "1" * 60 +JOURNALD = """nginx.service +ssh.service +postgresql@16-main.service +session-3.scope +init.scope +""" +OPEN = f"""/proc/610/fd/7 /var/log/syslog +/proc/610/fd/8 /var/log/auth.log +/proc/620/fd/9 /var/log/journal/0123abcd/system.journal +/proc/900/fd/5 /var/log/nginx/access.log +/proc/900/fd/6 /var/log/nginx/error.log +/proc/901/fd/5 /var/log/nginx/access.log +/proc/700/fd/4 /var/log/old.log (deleted) +/proc/1100/fd/30 /var/lib/docker/containers/{CID}/{CID}-json.log +/proc/1100/fd/31 /var/lib/docker/containers/{OTHER}/local-logs/container.log +/proc/1400/fd/3 /usr/src/paperless/data/log/paperless.log +/proc/1500/fd/3 /srv/app/log with space.log +/proc/1500/fd/4 /srv/app/Local Storage/leveldb/000004.log +/proc/1500/fd/5 /var/lib/rocksdb/012345.log +""" +STAT = """/proc/610/fd/7 99000 1762250000 +/proc/610/fd/8 4100 1762250100 +/proc/900/fd/5 2048 1762250200 +/proc/900/fd/6 512 1762250300 +/proc/901/fd/5 2048 1762250200 +/proc/1100/fd/30 7000 1762250400 +/proc/1100/fd/31 300 1762250500 +/proc/1400/fd/3 0 1762250600 +""" +CGROUPS = f"""610 0::/system.slice/rsyslog.service +900 0::/system.slice/nginx.service +901 0::/system.slice/nginx.service +1100 0::/system.slice/docker.service +1400 0::/system.slice/docker-{CID}.scope +1500 0::/user.slice/user-1000.slice/session-3.scope +""" +COMM = """610 rsyslogd +900 nginx +901 nginx +1100 dockerd +1400 python3 +1500 app +""" + + +def _wire( + *, + journald: str | None = JOURNALD, + open_: str = OPEN, + stat: str = STAT, + cgroups: str = CGROUPS, + comm: str = COMM, + noise: str = "", +) -> str: + """The report as the command prints it, framed.""" + parts = [noise, "LOGS_BEGIN\n"] + if journald is not None: + parts += ["[journald]\n", journald] + parts += ["[open]\n", open_, "[stat]\n", stat, "[cgroups]\n", cgroups, "[comm]\n", comm] + parts.append("LOGS_END\n") + return "".join(parts) + + +def _sources(output: str | None = None) -> list: + return parse_log_sources(_wire() if output is None else output)[1] + + +def _by_key(sources: list) -> dict: + return {(s["kind"], s["container_id"], s["path"] or s["unit"]): s for s in sources} + + +class TestParse: + def test_journald_names_each_service_unit_once_without_its_suffix(self) -> None: + journald, sources = parse_log_sources(_wire()) + units = [s["unit"] for s in sources if s["kind"] == "journald"] + assert units == ["nginx", "postgresql@16-main", "ssh"] + assert journald is True + + def test_a_host_without_journald_says_so(self) -> None: + journald, sources = parse_log_sources(_wire(journald=None)) + assert journald is False + assert not [s for s in sources if s["kind"] == "journald"] + + def test_an_open_file_names_its_writer_size_and_last_write(self) -> None: + sources = _by_key(_sources()) + access = sources[("file", None, "/var/log/nginx/access.log")] + assert access == { + "kind": "file", + "path": "/var/log/nginx/access.log", + "unit": "nginx", + "container_id": None, + "process": "nginx", + "size": 2048, + "modified": "2025-11-04T09:56:40+00:00", + } + + def test_a_file_two_processes_hold_is_one_source(self) -> None: + paths = [s["path"] for s in _sources() if s["kind"] == "file"] + assert paths.count("/var/log/nginx/access.log") == 1 + + def test_a_file_without_a_suffix_under_var_log_counts(self) -> None: + sources = _by_key(_sources()) + assert sources[("file", None, "/var/log/syslog")]["unit"] == "rsyslog" + + def test_a_file_a_container_writes_carries_the_container_and_its_own_path(self) -> None: + sources = _by_key(_sources()) + paperless = sources[("file", CID, "/usr/src/paperless/data/log/paperless.log")] + assert paperless["unit"] is None + assert paperless["process"] == "python3" + assert paperless["size"] == 0 + + def test_the_engines_own_log_file_is_the_containers_stream(self) -> None: + sources = _by_key(_sources()) + json_file = sources[("container", CID, f"/var/lib/docker/containers/{CID}/{CID}-json.log")] + assert json_file["process"] == "dockerd" + assert json_file["size"] == 7000 + local = sources[ + ("container", OTHER, f"/var/lib/docker/containers/{OTHER}/local-logs/container.log") + ] + assert local["modified"] == "2025-11-04T10:01:40+00:00" + assert not [s for s in _sources() if "docker" in (s["unit"] or "")] + + def test_journal_files_and_deleted_files_are_not_sources(self) -> None: + paths = [s["path"] or "" for s in _sources()] + assert not [p for p in paths if "/var/log/journal/" in p or p.startswith("/var/log/old")] + + def test_a_databases_write_ahead_log_is_not_a_source(self) -> None: + # LevelDB and RocksDB name theirs 000004.log; it holds records, not lines. + paths = [s["path"] or "" for s in _sources()] + assert not [p for p in paths if p.endswith(("000004.log", "012345.log"))] + + def test_a_path_with_a_space_survives(self) -> None: + sources = _by_key(_sources()) + spaced = sources[("file", None, "/srv/app/log with space.log")] + # The writer runs in a login session, not in a service; stat did not answer. + assert spaced["unit"] is None + assert spaced["process"] == "app" + assert spaced["size"] is None + assert spaced["modified"] is None + + def test_sources_come_sorted_by_kind_then_container_then_name(self) -> None: + sources = _sources() + order = [(s["kind"], s["container_id"] or "", s["path"] or s["unit"]) for s in sources] + assert order == sorted(order) + + def test_noise_before_the_frame_is_ignored(self) -> None: + assert _sources(_wire(noise="Last login: today\r\n\x1b[0m")) + + def test_a_host_without_gnu_find_raises_rather_than_reading_as_no_files(self) -> None: + # busybox find has neither -lname nor -printf: no files would read as none. + with pytest.raises(LogSourcesUnavailable, match="find"): + parse_log_sources("LOGS_BEGIN\n[journald]\nnginx.service\n[no-find]\nLOGS_END\n") + + def test_a_report_cut_short_raises_rather_than_reading_as_fewer_logs(self) -> None: + cut = _wire().replace("LOGS_END\n", "") + with pytest.raises(ValueError, match="log source report"): + parse_log_sources(cut) + + +class _Host(LogSourcesMixin): + """Carries the command the way a driver does; *answers* per privilege.""" + + def __init__(self, answers: dict[bool, str]) -> None: + self.answers = answers + self.calls: list[tuple[str, bool]] = [] + + def _run_log_sources_command(self, command: str, *, privileged: bool) -> str: + self.calls.append((command, privileged)) + return self.answers[privileged] + + +class TestMixin: + def test_it_reads_as_root_in_one_sh_call(self) -> None: + host = _Host({True: _wire()}) + reading = host.get_log_sources() + assert reading["attributed"] is True + command, privileged = host.calls[0] + assert privileged is True + assert command.startswith("sh -c ") + assert len(host.calls) == 1 + + def test_without_root_it_reads_again_and_says_so(self) -> None: + host = _Host({True: "sudo: a password is required\n", False: _wire()}) + reading = host.get_log_sources() + assert reading["attributed"] is False + assert [privileged for _, privileged in host.calls] == [True, False] + assert reading["sources"] + + def test_a_host_without_gnu_find_is_not_read_again_unprivileged(self) -> None: + host = _Host({True: "LOGS_BEGIN\n[no-find]\nLOGS_END\n"}) + with pytest.raises(LogSourcesUnavailable): + host.get_log_sources() + assert len(host.calls) == 1 + + def test_a_host_that_answers_neither_way_raises(self) -> None: + host = _Host({True: "", False: "garbage"}) + with pytest.raises(ValueError): + host.get_log_sources() + + +class TestCommand: + def test_it_is_valid_posix_sh(self) -> None: + assert subprocess.run(["sh", "-n", "-c", LOG_SOURCES_COMMAND]).returncode == 0 + + @pytest.mark.skipif(shutil.which("find") is None, reason="needs find on this host") + def test_it_runs_here_and_parses(self, tmp_path) -> None: + log = tmp_path / "held.log" + log.write_text("one line\n") + with log.open() as held: + out = subprocess.run( + ["sh", "-c", LOG_SOURCES_COMMAND], capture_output=True, text=True, timeout=60 + ).stdout + assert held.readable() + # This test's own process holds the file, so it must be among the sources. + mine = [s for s in _sources(out) if s["path"] == str(log)] + assert mine and mine[0]["size"] == len("one line\n") -- 2.54.0 From e622b26d42b368915e688b5c1e0afa46a6801f8b Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Thu, 5 Nov 2026 00:34:20 +0100 Subject: [PATCH 2/2] refactor: keep the log source shapes in log_sources, not in models models.py is over the file size limit and may not grow (quality ratchet); the shapes sit with their mixin, as ScheduledJobDict does. Refs NetOrk/netork#2049 --- napalm_device_types/__init__.py | 4 ++++ napalm_device_types/log_sources.py | 35 ++++++++++++++++++++++++++++-- napalm_device_types/models.py | 31 -------------------------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 6ec5dea..107d41e 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -169,6 +169,8 @@ from napalm_device_types.local_accounts import ( ) from napalm_device_types.log_sources import ( LOG_SOURCES_COMMAND, + LogSourceDict, + LogSourcesDict, LogSourcesMixin, LogSourcesUnavailable, parse_log_sources, @@ -406,6 +408,8 @@ __all__ = [ "ListeningSocketsMixin", "ListeningSocketsUnavailable", "LOG_SOURCES_COMMAND", + "LogSourceDict", + "LogSourcesDict", "LogSourcesMixin", "LogSourcesUnavailable", "parse_log_sources", diff --git a/napalm_device_types/log_sources.py b/napalm_device_types/log_sources.py index 99a60cf..90f2fbf 100644 --- a/napalm_device_types/log_sources.py +++ b/napalm_device_types/log_sources.py @@ -33,12 +33,43 @@ from __future__ import annotations import re from datetime import UTC, datetime from shlex import quote -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypedDict from napalm_device_types.listening import _cgroup_paths, _container, _sections, _unit -from napalm_device_types.models import LogSourceDict, LogSourcesDict from napalm_device_types.terminal import strip_terminal_codes + +class LogSourceDict(TypedDict): + """A log the host writes: a unit's journal, a file a process holds open, or a + container's stream as the engine keeps it. + + ``path`` is as its writer sees it: a file a container writes is named inside + that container. Whether anything reads the log is the consumer's question. + """ + + kind: str # "journald", "file" or "container" + path: str | None # the file; None for a journal + unit: str | None # the journal's unit, or the file writer's service; without ".service" + container_id: str | None # the full container ID: the writer's, or the stream's + process: str | None # the first process holding the file + size: int | None # bytes; None for a journal, or when the file could not be read + modified: str | None # the last write, ISO 8601 UTC; None as for size + + +class LogSourcesDict(TypedDict): + """What ``LogSourcesMixin.get_log_sources`` read. + + ``attributed`` is False when the reading ran without root: only the login + user's own processes show their files then, so a missing log means "not + told", not "not there". ``journald`` is False on a host without systemd's + journal, where no unit has one. + """ + + attributed: bool + journald: bool + sources: list[LogSourceDict] + + _BEGIN = "LOGS_BEGIN" _END = "LOGS_END" diff --git a/napalm_device_types/models.py b/napalm_device_types/models.py index 32b27e2..b2d4302 100644 --- a/napalm_device_types/models.py +++ b/napalm_device_types/models.py @@ -406,37 +406,6 @@ class ListeningSocketsDict(TypedDict): sockets: list[ListeningSocketDict] -class LogSourceDict(TypedDict): - """A log the host writes: a unit's journal, a file a process holds open, or a - container's stream as the engine keeps it. - - ``path`` is as its writer sees it: a file a container writes is named inside - that container. Whether anything reads the log is the consumer's question. - """ - - kind: str # "journald", "file" or "container" - path: str | None # the file; None for a journal - unit: str | None # the journal's unit, or the file writer's service; without ".service" - container_id: str | None # the full container ID: the writer's, or the stream's - process: str | None # the first process holding the file - size: int | None # bytes; None for a journal, or when the file could not be read - modified: str | None # the last write, ISO 8601 UTC; None as for size - - -class LogSourcesDict(TypedDict): - """What ``LogSourcesMixin.get_log_sources`` read. - - ``attributed`` is False when the reading ran without root: only the login - user's own processes show their files then, so a missing log means "not - told", not "not there". ``journald`` is False on a host without systemd's - journal, where no unit has one. - """ - - attributed: bool - journald: bool - sources: list[LogSourceDict] - - class NetfilterMatchDict(TypedDict): """One condition of a netfilter rule. -- 2.54.0