fix(log-sources): attribute a file only to the process writing it #96
No files matched your search
@@ -7,6 +7,14 @@ recorded here; their history is the git log.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
- `get_log_sources()` names as a file's writer only a process that holds it open for writing.
|
||||
A log reader -- CrowdSec, Promtail, rsyslog's `imfile`, a `tail -F` -- was given the file
|
||||
it reads as if it wrote it. A file only readers hold is now listed without a producer
|
||||
(`unit` and `process` None); one only a reader inside a container holds is left out, as its
|
||||
path is that container's view of a mount. The access mode comes from
|
||||
`/proc/<pid>/fdinfo/<fd>`, read in the same command (NetOrk/netork#2092).
|
||||
|
||||
## [4.31.0] - 2026-11-06
|
||||
|
||||
### Added
|
||||
|
||||
@@ -14,10 +14,16 @@ whose path is the container's own. The engine's own file of a container's
|
||||
output (``/var/lib/docker/containers/<id>/…``) is that container's stream, not
|
||||
a file of ``dockerd``.
|
||||
|
||||
**Who writes it.** ``/proc/<pid>/cgroup`` says which systemd service or
|
||||
container the first process holding a file runs in, as for listening sockets
|
||||
(:mod:`napalm_device_types.listening`). journald's own files, files already
|
||||
deleted and a database's write-ahead log (LevelDB's ``000004.log``) are left out.
|
||||
**Who writes it.** Only a process holding the file open for writing writes
|
||||
it: the access mode in ``/proc/<pid>/fdinfo/<fd>``'s ``flags`` is ``O_WRONLY`` or
|
||||
``O_RDWR``. A reader -- CrowdSec, Promtail, rsyslog's ``imfile``, a ``tail -F``
|
||||
-- holds it too, and is no producer (NetOrk/netork#2092). ``/proc/<pid>/cgroup``
|
||||
says which systemd service or container the first writer runs in, as for
|
||||
listening sockets (:mod:`napalm_device_types.listening`). A file only readers
|
||||
hold is listed without a producer; one only readers in a container hold is left
|
||||
out, its path being the reader's view of a mount. journald's own files, files
|
||||
already deleted and a database's write-ahead log (LevelDB's ``000004.log``) are
|
||||
left out.
|
||||
|
||||
**Where they go** is read in the same round trip
|
||||
(:mod:`napalm_device_types.log_forwarding`, NetOrk/netork#2053): whether
|
||||
@@ -58,7 +64,7 @@ class LogSourceDict(TypedDict):
|
||||
path: str | None # the file; None for a journal
|
||||
unit: str | None # the journal's unit, or the file writer's service; without ".service"
|
||||
container_id: str | None # the full container ID: the writer's, or the stream's
|
||||
process: str | None # the first process holding the file
|
||||
process: str | None # the first process writing the file; None when only readers hold it
|
||||
size: int | None # bytes; None for a journal, or when the file could not be read
|
||||
modified: str | None # the last write, ISO 8601 UTC; None as for size
|
||||
|
||||
@@ -98,6 +104,8 @@ LOG_SOURCES_COMMAND = (
|
||||
"echo '[open]'; printf '%s\\n' \"$o\"; "
|
||||
"echo '[stat]'; printf '%s\\n' \"$o\" | cut -d' ' -f1 "
|
||||
"| xargs -r stat -L -c '%n %s %Y' 2>/dev/null; "
|
||||
"echo '[flags]'; printf '%s\\n' \"$o\" | cut -d' ' -f1 | sed 's|/fd/|/fdinfo/|' "
|
||||
"| xargs -r grep -H '^flags:' 2>/dev/null; "
|
||||
"pids=$(printf '%s\\n' \"$o\" | cut -d/ -f3 | sort -un); "
|
||||
"echo '[cgroups]'; for p in $pids; do sed \"s|^|$p |\" /proc/$p/cgroup 2>/dev/null; done; "
|
||||
"echo '[comm]'; for p in $pids; do c=$(cat /proc/$p/comm 2>/dev/null) && echo \"$p $c\"; done; "
|
||||
@@ -114,6 +122,10 @@ LOG_SOURCES_COMMAND = (
|
||||
|
||||
#: ``/proc/900/fd/5`` -- the process and its descriptor.
|
||||
_FD_RE = re.compile(r"^/proc/(\d+)/fd/\d+$")
|
||||
#: ``/proc/900/fdinfo/5:flags:<tab>0102001`` -- the descriptor's open flags, octal.
|
||||
_FLAGS_RE = re.compile(r"^/proc/(\d+)/fdinfo/(\d+):flags:\s*([0-7]+)$")
|
||||
#: The access mode bits of the open flags: 0 is O_RDONLY, 1 O_WRONLY, 2 O_RDWR.
|
||||
_ACCESS_MODE = 0o3
|
||||
#: The engine's file of a container's output: json-file and local drivers.
|
||||
_STREAM_RE = re.compile(r"^/var/lib/docker/containers/([0-9a-f]{64})/")
|
||||
#: Files that are no log to read: journald's store, a file already deleted, and
|
||||
@@ -174,39 +186,83 @@ def _journals(units: list[str]) -> list[LogSourceDict]:
|
||||
|
||||
|
||||
def _held(lines: list[str]) -> list[tuple[int, str, str]]:
|
||||
"""(pid, descriptor, target) for every log a process holds, by pid."""
|
||||
"""(pid, descriptor, target) for every log file a process holds, by pid."""
|
||||
held = []
|
||||
for line in lines:
|
||||
fd, _, target = line.partition(" ")
|
||||
match = _FD_RE.match(fd)
|
||||
if match and target and not _NOT_A_SOURCE.search(target):
|
||||
if match and target.startswith("/") and not _NOT_A_SOURCE.search(target):
|
||||
held.append((int(match.group(1)), fd, target))
|
||||
return sorted(held)
|
||||
|
||||
|
||||
def _writes(lines: list[str]) -> dict[str, bool]:
|
||||
"""Whether each descriptor is open for writing, by its ``/proc/<pid>/fd/<fd>``."""
|
||||
writes: dict[str, bool] = {}
|
||||
for line in lines:
|
||||
match = _FLAGS_RE.match(line)
|
||||
if match:
|
||||
pid, fd, flags = match.groups()
|
||||
writes[f"/proc/{pid}/fd/{fd}"] = int(flags, 8) & _ACCESS_MODE != 0
|
||||
return writes
|
||||
|
||||
|
||||
_Key = tuple[str, str | None, str]
|
||||
|
||||
|
||||
def _holders(lines: list[str], paths: dict[int, str]) -> dict[_Key, list[tuple[int, str]]]:
|
||||
"""The (pid, descriptor) pairs holding each log, by kind, container and path."""
|
||||
holders: dict[_Key, list[tuple[int, str]]] = {}
|
||||
for pid, fd, target in _held(lines):
|
||||
stream = _STREAM_RE.match(target)
|
||||
kind, container = (
|
||||
("container", stream.group(1)) if stream else ("file", _container(paths.get(pid)))
|
||||
)
|
||||
holders.setdefault((kind, container, target), []).append((pid, fd))
|
||||
return holders
|
||||
|
||||
|
||||
def _writer(held: list[tuple[int, str]], writes: dict[str, bool] | None) -> int | None:
|
||||
"""The first process writing; the first holding for a report without flags,
|
||||
as an older command printed it."""
|
||||
if writes is None:
|
||||
return held[0][0]
|
||||
return next((pid for pid, fd in held if writes.get(fd)), None)
|
||||
|
||||
|
||||
def _producer(
|
||||
kind: str, writer: int | None, paths: dict[int, str], names: dict[int, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""The writer's service and name; a stream's service is its container."""
|
||||
if writer is None:
|
||||
return None, None
|
||||
return (_unit(paths.get(writer)) if kind == "file" else None), names.get(writer)
|
||||
|
||||
|
||||
def _files(sections: dict[str, list[str]]) -> list[LogSourceDict]:
|
||||
stats = _stats(sections.get("stat", []))
|
||||
paths = _cgroup_paths(sections.get("cgroups", []))
|
||||
names = _names(sections.get("comm", []))
|
||||
found: dict[tuple[str, str | None, str], LogSourceDict] = {}
|
||||
for pid, fd, target in _held(sections.get("open", [])):
|
||||
stream = _STREAM_RE.match(target)
|
||||
cgroup = paths.get(pid)
|
||||
kind, container = ("container", stream.group(1)) if stream else ("file", _container(cgroup))
|
||||
source = found.get((kind, container, target))
|
||||
if source is None:
|
||||
source = found[(kind, container, target)] = {
|
||||
writes = _writes(sections["flags"]) if "flags" in sections else None
|
||||
found: list[LogSourceDict] = []
|
||||
for (kind, container, target), held in _holders(sections.get("open", []), paths).items():
|
||||
writer = _writer(held, writes)
|
||||
if writer is None and kind == "file" and container is not None:
|
||||
continue # a reader's view of a mount, inside its container
|
||||
unit, process = _producer(kind, writer, paths, names)
|
||||
size, modified = next((stats[fd] for _, fd in held if fd in stats), (None, None))
|
||||
found.append(
|
||||
{
|
||||
"kind": kind,
|
||||
"path": target,
|
||||
"unit": None if stream else _unit(cgroup),
|
||||
"unit": unit,
|
||||
"container_id": container,
|
||||
"process": names.get(pid),
|
||||
"size": None,
|
||||
"modified": None,
|
||||
"process": process,
|
||||
"size": size,
|
||||
"modified": modified,
|
||||
}
|
||||
if source["size"] is None and fd in stats:
|
||||
source["size"], source["modified"] = stats[fd]
|
||||
return list(found.values())
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def _order(source: LogSourceDict) -> tuple[str, str, str]:
|
||||
|
||||
@@ -45,6 +45,35 @@ OPEN = f"""/proc/610/fd/7 /var/log/syslog
|
||||
/proc/1500/fd/3 /srv/app/log with space.log
|
||||
/proc/1500/fd/4 /srv/app/Local Storage/leveldb/000004.log
|
||||
/proc/1500/fd/5 /var/lib/rocksdb/012345.log
|
||||
/proc/300/fd/11 /var/log/nginx/access.log
|
||||
/proc/300/fd/12 /mnt/docroot/data/log/paperless.log
|
||||
/proc/300/fd/13 socket:[48213]
|
||||
/proc/300/fd/14 pipe:[48214]
|
||||
/proc/1600/fd/6 /var/log/host/syslog
|
||||
"""
|
||||
#: Each descriptor's ``flags`` from ``/proc/<pid>/fdinfo/<fd>``, octal: the access
|
||||
#: mode (``& 3``) is 1 or 2 for a writer -- 0102001 is O_WRONLY|O_APPEND|O_LARGEFILE
|
||||
#: -- and 0 for a reader (0100000, O_RDONLY|O_LARGEFILE). CrowdSec (300) reads
|
||||
#: nginx's log and a log on a mount; a promtail in a container (1600) reads the
|
||||
#: host's syslog as its own /var/log/host/syslog. 1500 opens its log O_RDWR.
|
||||
FLAGS = """/proc/610/fdinfo/7:flags:\t0102001
|
||||
/proc/610/fdinfo/8:flags:\t0102001
|
||||
/proc/620/fdinfo/9:flags:\t02100002
|
||||
/proc/900/fdinfo/5:flags:\t0102001
|
||||
/proc/900/fdinfo/6:flags:\t0102001
|
||||
/proc/901/fdinfo/5:flags:\t0102001
|
||||
/proc/700/fdinfo/4:flags:\t0102001
|
||||
/proc/1100/fdinfo/30:flags:\t0102001
|
||||
/proc/1100/fdinfo/31:flags:\t0102001
|
||||
/proc/1400/fdinfo/3:flags:\t0102001
|
||||
/proc/1500/fdinfo/3:flags:\t0100002
|
||||
/proc/1500/fdinfo/4:flags:\t0100002
|
||||
/proc/1500/fdinfo/5:flags:\t0100002
|
||||
/proc/300/fdinfo/11:flags:\t0100000
|
||||
/proc/300/fdinfo/12:flags:\t0100000
|
||||
/proc/300/fdinfo/13:flags:\t02004002
|
||||
/proc/300/fdinfo/14:flags:\t02004001
|
||||
/proc/1600/fdinfo/6:flags:\t0100000
|
||||
"""
|
||||
STAT = """/proc/610/fd/7 99000 1762250000
|
||||
/proc/610/fd/8 4100 1762250100
|
||||
@@ -54,6 +83,8 @@ STAT = """/proc/610/fd/7 99000 1762250000
|
||||
/proc/1100/fd/30 7000 1762250400
|
||||
/proc/1100/fd/31 300 1762250500
|
||||
/proc/1400/fd/3 0 1762250600
|
||||
/proc/300/fd/11 2048 1762250200
|
||||
/proc/300/fd/12 81920 1762250700
|
||||
"""
|
||||
CGROUPS = f"""610 0::/system.slice/rsyslog.service
|
||||
900 0::/system.slice/nginx.service
|
||||
@@ -61,6 +92,8 @@ CGROUPS = f"""610 0::/system.slice/rsyslog.service
|
||||
1100 0::/system.slice/docker.service
|
||||
1400 0::/system.slice/docker-{CID}.scope
|
||||
1500 0::/user.slice/user-1000.slice/session-3.scope
|
||||
300 0::/system.slice/crowdsec.service
|
||||
1600 0::/system.slice/docker-{OTHER}.scope
|
||||
"""
|
||||
COMM = """610 rsyslogd
|
||||
900 nginx
|
||||
@@ -68,6 +101,8 @@ COMM = """610 rsyslogd
|
||||
1100 dockerd
|
||||
1400 python3
|
||||
1500 app
|
||||
300 crowdsec
|
||||
1600 promtail
|
||||
"""
|
||||
|
||||
|
||||
@@ -78,6 +113,7 @@ def _wire(
|
||||
stat: str = STAT,
|
||||
cgroups: str = CGROUPS,
|
||||
comm: str = COMM,
|
||||
flags: str | None = FLAGS,
|
||||
noise: str = "",
|
||||
forwarding: str = "",
|
||||
) -> str:
|
||||
@@ -86,6 +122,8 @@ def _wire(
|
||||
if journald is not None:
|
||||
parts += ["[journald]\n", journald]
|
||||
parts += ["[open]\n", open_, "[stat]\n", stat, "[cgroups]\n", cgroups, "[comm]\n", comm]
|
||||
if flags is not None:
|
||||
parts += ["[flags]\n", flags]
|
||||
parts += [forwarding, "LOGS_END\n"]
|
||||
return "".join(parts)
|
||||
|
||||
@@ -186,6 +224,55 @@ class TestParse:
|
||||
parse_log_sources(cut)
|
||||
|
||||
|
||||
class TestWriters:
|
||||
"""A file is attributed to the process writing it, not to one reading it
|
||||
(NetOrk/netork#2092): CrowdSec, Promtail or a ``tail -F`` holding a log is
|
||||
no producer of it."""
|
||||
|
||||
def test_a_reader_with_the_lower_pid_does_not_take_the_writers_file(self) -> None:
|
||||
access = _by_key(_sources())[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == ("nginx", "nginx")
|
||||
|
||||
def test_a_file_only_readers_hold_is_listed_without_a_producer(self) -> None:
|
||||
paperless = _by_key(_sources())[("file", None, "/mnt/docroot/data/log/paperless.log")]
|
||||
assert paperless == {
|
||||
"kind": "file",
|
||||
"path": "/mnt/docroot/data/log/paperless.log",
|
||||
"unit": None,
|
||||
"container_id": None,
|
||||
"process": None,
|
||||
"size": 81920,
|
||||
"modified": "2025-11-04T10:05:00+00:00",
|
||||
}
|
||||
|
||||
def test_a_file_open_for_reading_and_writing_is_written(self) -> None:
|
||||
spaced = _by_key(_sources())[("file", None, "/srv/app/log with space.log")]
|
||||
assert spaced["process"] == "app"
|
||||
|
||||
def test_a_file_only_a_reader_in_a_container_holds_is_left_out(self) -> None:
|
||||
# Its path is the reader's view of a mount: it names nothing on the host,
|
||||
# and the container does not write it.
|
||||
assert not [s for s in _sources() if s["path"] == "/var/log/host/syslog"]
|
||||
|
||||
def test_sockets_and_pipes_are_no_sources(self) -> None:
|
||||
assert not [s for s in _sources() if not (s["path"] or "/").startswith("/")]
|
||||
|
||||
def test_a_deleted_file_stays_out_though_it_is_written(self) -> None:
|
||||
assert not [s for s in _sources() if (s["path"] or "").startswith("/var/log/old")]
|
||||
|
||||
def test_a_holder_whose_flags_did_not_answer_is_no_writer(self) -> None:
|
||||
# The process ended between the listing and the read of its fdinfo.
|
||||
flags = FLAGS.replace("/proc/900/fdinfo/5:flags:\t0102001\n", "")
|
||||
flags = flags.replace("/proc/901/fdinfo/5:flags:\t0102001\n", "")
|
||||
access = _by_key(_sources(_wire(flags=flags)))[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == (None, None)
|
||||
|
||||
def test_a_report_without_flags_names_the_first_holder_as_before(self) -> None:
|
||||
# An older command printed no [flags]: who writes cannot be told apart.
|
||||
access = _by_key(_sources(_wire(flags=None)))[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == ("crowdsec", "crowdsec")
|
||||
|
||||
|
||||
#: rsyslog running, forwarding to Graylog, its config prefixed so that no line reads as a section.
|
||||
FORWARDING = """[rsyslog]
|
||||
running
|
||||
@@ -276,6 +363,21 @@ class TestCommand:
|
||||
def test_it_is_valid_posix_sh(self) -> None:
|
||||
assert subprocess.run(["sh", "-n", "-c", LOG_SOURCES_COMMAND]).returncode == 0
|
||||
|
||||
@pytest.mark.skipif(shutil.which("find") is None, reason="needs find on this host")
|
||||
def test_here_a_file_held_for_writing_has_a_producer_and_one_held_for_reading_none(
|
||||
self, tmp_path
|
||||
) -> None:
|
||||
written, read = tmp_path / "written.log", tmp_path / "read.log"
|
||||
read.write_text("one line\n")
|
||||
with written.open("a") as w, read.open() as r:
|
||||
out = subprocess.run(
|
||||
["sh", "-c", LOG_SOURCES_COMMAND], capture_output=True, text=True, timeout=60
|
||||
).stdout
|
||||
assert w.writable() and r.readable()
|
||||
sources = {s["path"]: s for s in _sources(out)}
|
||||
assert sources[str(written)]["process"] is not None
|
||||
assert sources[str(read)]["process"] is None
|
||||
|
||||
@pytest.mark.skipif(shutil.which("find") is None, reason="needs find on this host")
|
||||
def test_it_runs_here_and_parses(self, tmp_path) -> None:
|
||||
log = tmp_path / "held.log"
|
||||
|
||||
Reference in new issue
Block a user