fix(log-sources): attribute a file only to the process writing it
driver / standard (pull_request) Successful in 16s
driver / lint (pull_request) Successful in 14s
driver / types (pull_request) Successful in 1m8s
driver / test (3.13) (pull_request) Successful in 1m8s
driver / test (3.12) (pull_request) Successful in 1m15s
driver / lowest (pull_request) Successful in 40s
driver / build (pull_request) Successful in 29s
driver / security (pull_request) Successful in 59s
CI / driver (pull_request) Successful in 2m25s
driver / publish (pull_request) Skipped
driver / standard (pull_request) Successful in 16s
driver / lint (pull_request) Successful in 14s
driver / types (pull_request) Successful in 1m8s
driver / test (3.13) (pull_request) Successful in 1m8s
driver / test (3.12) (pull_request) Successful in 1m15s
driver / lowest (pull_request) Successful in 40s
driver / build (pull_request) Successful in 29s
driver / security (pull_request) Successful in 59s
CI / driver (pull_request) Successful in 2m25s
driver / publish (pull_request) Skipped
A process that holds a log open only to read it -- CrowdSec's acquisition, Promtail, rsyslog imfile, a tail -F -- was named as the file's writer whenever it had the lower pid. The command now reads each descriptor's flags from /proc/<pid>/fdinfo in the same round trip, and only an O_WRONLY or O_RDWR holder is the producer. A file only readers hold stays listed without one; a file only a reader inside a container holds is its view of a mount and is left out. Refs: NetOrk/netork#2092
This commit is contained in:
1 parent
252ea3b81c
commit
98d1dbef4c
3 files changed
+188
-22
No files matched your search
@@ -45,6 +45,35 @@ OPEN = f"""/proc/610/fd/7 /var/log/syslog
|
||||
/proc/1500/fd/3 /srv/app/log with space.log
|
||||
/proc/1500/fd/4 /srv/app/Local Storage/leveldb/000004.log
|
||||
/proc/1500/fd/5 /var/lib/rocksdb/012345.log
|
||||
/proc/300/fd/11 /var/log/nginx/access.log
|
||||
/proc/300/fd/12 /mnt/docroot/data/log/paperless.log
|
||||
/proc/300/fd/13 socket:[48213]
|
||||
/proc/300/fd/14 pipe:[48214]
|
||||
/proc/1600/fd/6 /var/log/host/syslog
|
||||
"""
|
||||
#: Each descriptor's ``flags`` from ``/proc/<pid>/fdinfo/<fd>``, octal: the access
|
||||
#: mode (``& 3``) is 1 or 2 for a writer -- 0102001 is O_WRONLY|O_APPEND|O_LARGEFILE
|
||||
#: -- and 0 for a reader (0100000, O_RDONLY|O_LARGEFILE). CrowdSec (300) reads
|
||||
#: nginx's log and a log on a mount; a promtail in a container (1600) reads the
|
||||
#: host's syslog as its own /var/log/host/syslog. 1500 opens its log O_RDWR.
|
||||
FLAGS = """/proc/610/fdinfo/7:flags:\t0102001
|
||||
/proc/610/fdinfo/8:flags:\t0102001
|
||||
/proc/620/fdinfo/9:flags:\t02100002
|
||||
/proc/900/fdinfo/5:flags:\t0102001
|
||||
/proc/900/fdinfo/6:flags:\t0102001
|
||||
/proc/901/fdinfo/5:flags:\t0102001
|
||||
/proc/700/fdinfo/4:flags:\t0102001
|
||||
/proc/1100/fdinfo/30:flags:\t0102001
|
||||
/proc/1100/fdinfo/31:flags:\t0102001
|
||||
/proc/1400/fdinfo/3:flags:\t0102001
|
||||
/proc/1500/fdinfo/3:flags:\t0100002
|
||||
/proc/1500/fdinfo/4:flags:\t0100002
|
||||
/proc/1500/fdinfo/5:flags:\t0100002
|
||||
/proc/300/fdinfo/11:flags:\t0100000
|
||||
/proc/300/fdinfo/12:flags:\t0100000
|
||||
/proc/300/fdinfo/13:flags:\t02004002
|
||||
/proc/300/fdinfo/14:flags:\t02004001
|
||||
/proc/1600/fdinfo/6:flags:\t0100000
|
||||
"""
|
||||
STAT = """/proc/610/fd/7 99000 1762250000
|
||||
/proc/610/fd/8 4100 1762250100
|
||||
@@ -54,6 +83,8 @@ STAT = """/proc/610/fd/7 99000 1762250000
|
||||
/proc/1100/fd/30 7000 1762250400
|
||||
/proc/1100/fd/31 300 1762250500
|
||||
/proc/1400/fd/3 0 1762250600
|
||||
/proc/300/fd/11 2048 1762250200
|
||||
/proc/300/fd/12 81920 1762250700
|
||||
"""
|
||||
CGROUPS = f"""610 0::/system.slice/rsyslog.service
|
||||
900 0::/system.slice/nginx.service
|
||||
@@ -61,6 +92,8 @@ CGROUPS = f"""610 0::/system.slice/rsyslog.service
|
||||
1100 0::/system.slice/docker.service
|
||||
1400 0::/system.slice/docker-{CID}.scope
|
||||
1500 0::/user.slice/user-1000.slice/session-3.scope
|
||||
300 0::/system.slice/crowdsec.service
|
||||
1600 0::/system.slice/docker-{OTHER}.scope
|
||||
"""
|
||||
COMM = """610 rsyslogd
|
||||
900 nginx
|
||||
@@ -68,6 +101,8 @@ COMM = """610 rsyslogd
|
||||
1100 dockerd
|
||||
1400 python3
|
||||
1500 app
|
||||
300 crowdsec
|
||||
1600 promtail
|
||||
"""
|
||||
|
||||
|
||||
@@ -78,6 +113,7 @@ def _wire(
|
||||
stat: str = STAT,
|
||||
cgroups: str = CGROUPS,
|
||||
comm: str = COMM,
|
||||
flags: str | None = FLAGS,
|
||||
noise: str = "",
|
||||
forwarding: str = "",
|
||||
) -> str:
|
||||
@@ -86,6 +122,8 @@ def _wire(
|
||||
if journald is not None:
|
||||
parts += ["[journald]\n", journald]
|
||||
parts += ["[open]\n", open_, "[stat]\n", stat, "[cgroups]\n", cgroups, "[comm]\n", comm]
|
||||
if flags is not None:
|
||||
parts += ["[flags]\n", flags]
|
||||
parts += [forwarding, "LOGS_END\n"]
|
||||
return "".join(parts)
|
||||
|
||||
@@ -186,6 +224,55 @@ class TestParse:
|
||||
parse_log_sources(cut)
|
||||
|
||||
|
||||
class TestWriters:
|
||||
"""A file is attributed to the process writing it, not to one reading it
|
||||
(NetOrk/netork#2092): CrowdSec, Promtail or a ``tail -F`` holding a log is
|
||||
no producer of it."""
|
||||
|
||||
def test_a_reader_with_the_lower_pid_does_not_take_the_writers_file(self) -> None:
|
||||
access = _by_key(_sources())[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == ("nginx", "nginx")
|
||||
|
||||
def test_a_file_only_readers_hold_is_listed_without_a_producer(self) -> None:
|
||||
paperless = _by_key(_sources())[("file", None, "/mnt/docroot/data/log/paperless.log")]
|
||||
assert paperless == {
|
||||
"kind": "file",
|
||||
"path": "/mnt/docroot/data/log/paperless.log",
|
||||
"unit": None,
|
||||
"container_id": None,
|
||||
"process": None,
|
||||
"size": 81920,
|
||||
"modified": "2025-11-04T10:05:00+00:00",
|
||||
}
|
||||
|
||||
def test_a_file_open_for_reading_and_writing_is_written(self) -> None:
|
||||
spaced = _by_key(_sources())[("file", None, "/srv/app/log with space.log")]
|
||||
assert spaced["process"] == "app"
|
||||
|
||||
def test_a_file_only_a_reader_in_a_container_holds_is_left_out(self) -> None:
|
||||
# Its path is the reader's view of a mount: it names nothing on the host,
|
||||
# and the container does not write it.
|
||||
assert not [s for s in _sources() if s["path"] == "/var/log/host/syslog"]
|
||||
|
||||
def test_sockets_and_pipes_are_no_sources(self) -> None:
|
||||
assert not [s for s in _sources() if not (s["path"] or "/").startswith("/")]
|
||||
|
||||
def test_a_deleted_file_stays_out_though_it_is_written(self) -> None:
|
||||
assert not [s for s in _sources() if (s["path"] or "").startswith("/var/log/old")]
|
||||
|
||||
def test_a_holder_whose_flags_did_not_answer_is_no_writer(self) -> None:
|
||||
# The process ended between the listing and the read of its fdinfo.
|
||||
flags = FLAGS.replace("/proc/900/fdinfo/5:flags:\t0102001\n", "")
|
||||
flags = flags.replace("/proc/901/fdinfo/5:flags:\t0102001\n", "")
|
||||
access = _by_key(_sources(_wire(flags=flags)))[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == (None, None)
|
||||
|
||||
def test_a_report_without_flags_names_the_first_holder_as_before(self) -> None:
|
||||
# An older command printed no [flags]: who writes cannot be told apart.
|
||||
access = _by_key(_sources(_wire(flags=None)))[("file", None, "/var/log/nginx/access.log")]
|
||||
assert (access["unit"], access["process"]) == ("crowdsec", "crowdsec")
|
||||
|
||||
|
||||
#: rsyslog running, forwarding to Graylog, its config prefixed so that no line reads as a section.
|
||||
FORWARDING = """[rsyslog]
|
||||
running
|
||||
@@ -276,6 +363,21 @@ class TestCommand:
|
||||
def test_it_is_valid_posix_sh(self) -> None:
|
||||
assert subprocess.run(["sh", "-n", "-c", LOG_SOURCES_COMMAND]).returncode == 0
|
||||
|
||||
@pytest.mark.skipif(shutil.which("find") is None, reason="needs find on this host")
|
||||
def test_here_a_file_held_for_writing_has_a_producer_and_one_held_for_reading_none(
|
||||
self, tmp_path
|
||||
) -> None:
|
||||
written, read = tmp_path / "written.log", tmp_path / "read.log"
|
||||
read.write_text("one line\n")
|
||||
with written.open("a") as w, read.open() as r:
|
||||
out = subprocess.run(
|
||||
["sh", "-c", LOG_SOURCES_COMMAND], capture_output=True, text=True, timeout=60
|
||||
).stdout
|
||||
assert w.writable() and r.readable()
|
||||
sources = {s["path"]: s for s in _sources(out)}
|
||||
assert sources[str(written)]["process"] is not None
|
||||
assert sources[str(read)]["process"] is None
|
||||
|
||||
@pytest.mark.skipif(shutil.which("find") is None, reason="needs find on this host")
|
||||
def test_it_runs_here_and_parses(self, tmp_path) -> None:
|
||||
log = tmp_path / "held.log"
|
||||
|
||||
Reference in new issue
Block a user