Merge pull request 'feat: the configuration that says which services run together (4.17.0)' (#47) from feat/service-clusters into main

tests/test_service_clusters.py

@@ -0,0 +1,491 @@
+"""get_service_clusters: the configuration that says which services run together.
+
+Two hosts that answer at one virtual IP may run the same service by accident.
+What proves a cluster is the configuration the members share: Patroni's scope
+and the members its DCS lists, adguardhome-sync's origin and replicas, the
+address a BunkerWeb worker takes its configuration from. Reading that is the
+same on every Linux host, so the command and its parse live here once, and a
+driver only carries the command across (netOrk #1125).
+
+The same configuration holds secrets -- Patroni's superuser password, the sync's
+AdGuard logins, a database URI -- so the command filters on the host: only
+allowlisted fields ever leave it. The tests below run the command for real to
+hold it to that.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from napalm_device_types.service_clusters import (
+    SERVICE_CLUSTERS_COMMAND,
+    ServiceClustersMixin,
+    parse_service_clusters,
+    service_clusters_command,
+)
+
+SECRET = "s3cr3t-do-not-leak"
+
+
+def _wire(*sections: str, systemd: bool = True, docker: str = "[docker]", noise: str = "") -> str:
+    head = "[systemd]" if systemd else "[no-systemd]"
+    body = "\n".join([head, *sections[:1], docker, *sections[1:]]) if sections else f"{head}\n{docker}"
+    return f"{noise}SVCC_BEGIN\n{body}\nSVCC_END\n"
+
+
+DB01_CLUSTER = {
+    "members": [
+        {"name": "db-01", "role": "leader", "state": "running", "host": "10.7.234.12", "port": 5432},
+        {"name": "db-02", "role": "sync_standby", "state": "streaming", "host": "10.7.234.11", "port": 5432, "lag": 0},
+    ],
+    "scope": "netork-db",
+}
+
+
+def _patroni(*records: str, unit: str = "patroni.service") -> str:
+    return "\n".join([f"[patroni {unit}]", *records])
+
+
+def _container(name: str, image: str, *env: str) -> str:
+    return "\n".join([f"[container {name}]", f"image {image}", *(f"env {e}" for e in env)])
+
+
+class TestPatroni:
+    def test_the_members_its_dcs_lists_with_their_roles(self):
+        reading = parse_service_clusters(
+            _wire(
+                _patroni(
+                    "active active",
+                    "config /etc/patroni/config.yml",
+                    "scope netork-db",
+                    "name db-01",
+                    "restapi 10.7.234.12:8008",
+                    "rest " + json.dumps(DB01_CLUSTER),
+                )
+            )
+        )
+
+        assert reading["patroni"] == [
+            {
+                "mechanism": "patroni",
+                "service": "patroni",
+                "source": "patroni",
+                "cluster": "netork-db",
+                "running": True,
+                "members": [
+                    {"name": "db-01", "address": "10.7.234.12", "port": 5432, "role": "leader",
+                     "state": "running", "this_host": True, "config_from": None},
+                    {"name": "db-02", "address": "10.7.234.11", "port": 5432, "role": "sync_standby",
+                     "state": "streaming", "this_host": False, "config_from": None},
+                ],
+            }
+        ]
+
+    def test_without_its_rest_api_it_is_itself_in_its_scope(self):
+        reading = parse_service_clusters(
+            _wire(
+                _patroni(
+                    "active active", "scope 'netork-db'  # the cluster", "name db-01",
+                    "restapi 10.7.234.12:8008", "rest-unreachable",
+                )
+            )
+        )
+
+        (cluster,) = reading["patroni"]
+        assert cluster["cluster"] == "netork-db"
+        assert cluster["members"] == [
+            {"name": "db-01", "address": "10.7.234.12", "port": None, "role": None,
+             "state": None, "this_host": True, "config_from": None}
+        ]
+
+    def test_an_unreadable_config_is_not_looked_at(self):
+        reading = parse_service_clusters(
+            _wire(_patroni("active active", "unreadable /etc/patroni/config.yml"))
+        )
+
+        assert reading["patroni"] is None
+
+    def test_a_config_without_a_scope_declares_no_cluster(self):
+        assert parse_service_clusters(_wire(_patroni("active active", "name db-01")))["patroni"] == []
+
+    def test_a_host_without_systemd_is_not_looked_at(self):
+        assert parse_service_clusters(_wire(systemd=False))["patroni"] is None
+
+    def test_no_patroni_is_looked_and_found_nothing(self):
+        assert parse_service_clusters(_wire())["patroni"] == []
+
+
+class TestAdguardhomeSync:
+    def test_the_origin_pushes_to_each_replica(self):
+        reading = parse_service_clusters(
+            _wire(
+                "",
+                _container(
+                    "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3",
+                    "ORIGIN_URL http://10.9.100.5:3000", "REPLICA1_URL http://10.9.100.6:3000",
+                    "REPLICA2_URL https://dns-03.example.org",
+                ),
+            )
+        )
+
+        (cluster,) = reading["adguardhome_sync"]
+        assert (cluster["mechanism"], cluster["service"], cluster["cluster"]) == (
+            "adguardhome-sync", "adguardhome", "10.9.100.5:3000",
+        )
+        assert [(m["address"], m["port"], m["role"], m["config_from"]) for m in cluster["members"]] == [
+            ("10.9.100.5", 3000, "origin", None),
+            ("10.9.100.6", 3000, "replica", 0),
+            ("dns-03.example.org", 443, "replica", 0),
+        ]
+        assert not any(m["this_host"] for m in cluster["members"])
+
+    def test_a_single_replica_url_counts(self):
+        reading = parse_service_clusters(
+            _wire("", _container("sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000",
+                                 "REPLICA_URL http://b:3000"))
+        )
+
+        assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == ["a", "b"]
+
+    def test_a_sync_configured_by_file_is_not_looked_at(self):
+        """Its YAML is not read: the cluster is there, unseen."""
+        reading = parse_service_clusters(_wire("", _container("sync", "bakito/adguardhome-sync")))
+
+        assert reading["adguardhome_sync"] is None
+
+
+class TestBunkerweb:
+    def test_a_worker_takes_its_configuration_from_the_manager_it_admits(self):
+        reading = parse_service_clusters(
+            _wire("", _container("bunkerweb", "bunkerity/bunkerweb@sha256:2026", "API_WHITELIST_IP 10.50.100.5"))
+        )
+
+        (cluster,) = reading["bunkerweb"]
+        assert (cluster["service"], cluster["cluster"]) == ("bunkerweb", "10.50.100.5")
+        assert cluster["members"] == [
+            {"name": None, "address": None, "port": 5000, "role": "worker", "state": None,
+             "this_host": True, "config_from": 1},
+            {"name": None, "address": "10.50.100.5", "port": None, "role": "manager", "state": None,
+             "this_host": False, "config_from": None},
+        ]
+
+    def test_the_default_whitelist_names_no_manager(self):
+        """127.0.0.0/8 and a docker network are BunkerWeb's default, not a peer."""
+        reading = parse_service_clusters(
+            _wire("", _container("bw", "bunkerity/bunkerweb:1.6", "API_WHITELIST_IP 127.0.0.0/8 10.20.30.0/24"))
+        )
+
+        assert reading["bunkerweb"] == []
+
+    def test_a_manager_names_its_instances(self):
+        reading = parse_service_clusters(
+            _wire("", _container("bw", "bunkerity/bunkerweb-all-in-one:1.6",
+                                 "BUNKERWEB_INSTANCES 10.50.100.6:5000 bw-03")),
+        )
+
+        (cluster,) = reading["bunkerweb"]
+        assert cluster["cluster"] is None  # the manager cannot name its own address
+        assert [(m["address"], m["port"], m["role"], m["this_host"], m["config_from"])
+                for m in cluster["members"]] == [
+            (None, None, "manager", True, None),
+            ("10.50.100.6", 5000, "worker", False, 0),
+            ("bw-03", 5000, "worker", False, 0),
+        ]
+
+    def test_a_manager_without_instances_is_no_cluster(self):
+        """proxy-01's all-in-one keeps its instances in its database."""
+        reading = parse_service_clusters(_wire("", _container("bw", "bunkerity/bunkerweb-all-in-one@sha256:2b34")))
+
+        assert reading["bunkerweb"] == []
+
+    def test_its_ui_and_scheduler_side_containers_are_not_instances(self):
+        reading = parse_service_clusters(
+            _wire("", _container("ui", "bunkerity/bunkerweb-ui:1.6", "API_WHITELIST_IP 10.0.0.9"))
+        )
+
+        assert reading["bunkerweb"] == []
+
+
+class TestTheReport:
+    def test_a_host_without_docker_runs_no_containers(self):
+        reading = parse_service_clusters(_wire(docker="[no-docker]"))
+
+        assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], [])
+
+    def test_a_docker_it_may_not_ask_is_not_looked_at(self):
+        reading = parse_service_clusters(_wire(docker="[docker-unreadable]"))
+
+        assert (reading["adguardhome_sync"], reading["bunkerweb"]) == (None, None)
+
+    def test_noise_before_the_report_is_ignored(self):
+        assert parse_service_clusters(_wire(noise="Last login: today\n"))["patroni"] == []
+
+    def test_a_report_cut_short_raises_without_quoting_it(self):
+        with pytest.raises(ValueError) as caught:
+            parse_service_clusters("SVCC_BEGIN\n[systemd]\nscope netork-db\n")
+
+        assert "netork-db" not in str(caught.value)
+
+
+# ---------------------------------------------------------------------------
+# The command itself, run against stubs and real files
+# ---------------------------------------------------------------------------
+
+PATRONI_YML = f"""scope: netork-db
+namespace: /service/
+name: db-01
+
+restapi:
+  listen: 10.7.234.12:8008
+  connect_address: 10.7.234.12:8008
+  authentication:
+    username: patroni
+    password: {SECRET}
+
+etcd3:
+  hosts:
+    - 10.7.234.12:2379
+    - 10.7.234.13:2379
+  password: {SECRET}
+
+postgresql:
+  connect_address: 10.7.234.12:5432
+  authentication:
+    superuser:
+      username: postgres
+      password: {SECRET}
+tags:
+  name: not-the-member-name
+"""
+
+
+def _awks() -> list:
+    found = []
+    for name, argv in (("mawk", ["mawk"]), ("busybox", ["busybox", "awk"]), ("gawk", ["gawk"])):
+        if shutil.which(argv[0]):
+            found.append(pytest.param(argv, id=name))
+    return found or [pytest.param(None, marks=pytest.mark.skip(reason="no awk to run"))]
+
+
+class _Host:
+    """A host as far as the command can tell: units, containers, a REST API, files."""
+
+    def __init__(self, root: Path, awk: list) -> None:
+        self.root = root
+        self.bin = root / "bin"
+        self.bin.mkdir()
+        self.state = root / "state"
+        self.state.mkdir()
+        self.booted = root / "run-systemd-system"
+        self.booted.mkdir()
+        (self.state / "units").write_text("")
+        (self.state / "containers").write_text("")
+        cat = shutil.which("cat")
+        self._stub(
+            "systemctl",
+            'case "$1" in\n'
+            f'  list-units) {cat} "$d/units" ;;\n'
+            f'  show) for last; do :; done; {cat} "$d/show-$last" ;;\n'
+            "esac\n",
+        )
+        self._stub(
+            "docker",
+            'case "$1" in\n'
+            f'  ps) {cat} "$d/containers" ;;\n'
+            '  inspect) for last; do :; done;\n'
+            '    case "$3" in *Config.Image*) '
+            f'{cat} "$d/image-$last" ;; *) {cat} "$d/env-$last" ;; esac ;;\n'
+            "esac\n",
+        )
+        self._stub("curl", f'for last; do :; done; {cat} "$d/rest" 2>/dev/null || exit 7\n')
+        for tool in ("sed", "head", "tr"):
+            if shutil.which(tool):
+                (self.bin / tool).symlink_to(shutil.which(tool))
+        wrapper = self.bin / "awk"
+        program = " ".join([shutil.which(awk[0]) or awk[0], *awk[1:]])
+        wrapper.write_text(f'#!/bin/sh\nexec {program} "$@"\n')
+        wrapper.chmod(0o755)
+
+    def _stub(self, name: str, body: str) -> None:
+        stub = self.bin / name
+        stub.write_text(f"#!/bin/sh\nd={self.state}\n{body}")
+        stub.chmod(0o755)
+
+    def unit(self, name: str, *properties: str) -> None:
+        with (self.state / "units").open("a") as units:
+            units.write(f"{name} loaded active running {name}\n")
+        (self.state / f"show-{name}").write_text("".join(f"{p}\n" for p in properties))
+
+    def container(self, name: str, image: str, *env: str) -> None:
+        with (self.state / "containers").open("a") as containers:
+            containers.write(f"{name}\n")
+        (self.state / f"image-{name}").write_text(f"{image}\n")
+        (self.state / f"env-{name}").write_text("".join(f"{e}\n" for e in env))
+
+    def rest(self, body: dict) -> None:
+        (self.state / "rest").write_text(json.dumps(body))
+
+    def file(self, relative: str, content: str) -> Path:
+        path = self.root / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content)
+        return path
+
+    def run(self, *, docker: bool = True) -> str:
+        command = service_clusters_command("docker" if docker else None)
+        assert command.count("/run/systemd/system") == 1
+        command = command.replace("/run/systemd/system", str(self.booted))
+        return subprocess.run(
+            ["/bin/sh", "-c", command],
+            capture_output=True,
+            text=True,
+            env={"PATH": str(self.bin)},
+            timeout=30,
+        ).stdout
+
+
+def _exec_start(*argv: str) -> str:
+    return f"ExecStart={{ path={argv[0]} ; argv[]={' '.join(argv)} ; ignore_errors=no ; pid=7 ; status=0/0 }}"
+
+
+@pytest.fixture(params=_awks())
+def host(request, tmp_path) -> _Host:
+    return _Host(tmp_path, request.param)
+
+
+class TestTheCommandOnAHost:
+    def test_patroni_as_on_db_01(self, host):
+        config = host.file("etc/patroni/config.yml", PATRONI_YML)
+        host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)))
+        host.rest(DB01_CLUSTER)
+
+        out = host.run()
+
+        assert SECRET not in out
+        assert "10.7.234.13" not in out  # the DCS endpoints stay on the host
+        assert "not-the-member-name" not in out
+        (cluster,) = parse_service_clusters(out)["patroni"]
+        assert cluster["cluster"] == "netork-db"
+        assert [(m["name"], m["role"], m["this_host"]) for m in cluster["members"]] == [
+            ("db-01", "leader", True),
+            ("db-02", "sync_standby", False),
+        ]
+
+    def test_patroni_whose_rest_api_does_not_answer(self, host):
+        config = host.file("etc/patroni/config.yml", PATRONI_YML)
+        host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)))
+
+        (cluster,) = parse_service_clusters(host.run())["patroni"]
+
+        assert [(m["name"], m["address"], m["role"]) for m in cluster["members"]] == [
+            ("db-01", "10.7.234.12", None)
+        ]
+
+    @pytest.mark.skipif(os.geteuid() == 0, reason="root reads a file of mode 000")
+    def test_a_patroni_config_it_cannot_read_is_not_looked_at(self, host):
+        config = host.file("etc/patroni/config.yml", PATRONI_YML)
+        config.chmod(0)
+        host.unit("patroni.service", "ActiveState=active", _exec_start("/usr/bin/patroni", str(config)))
+
+        out = host.run()
+
+        assert SECRET not in out
+        assert parse_service_clusters(out)["patroni"] is None
+
+    def test_the_sync_and_a_worker_without_their_secrets(self, host):
+        host.container(
+            "adguardhome-sync", "ghcr.io/bakito/adguardhome-sync:v0.9.3",
+            "ORIGIN_URL=http://10.9.100.5:3000", f"ORIGIN_PASSWORD={SECRET}",
+            f"REPLICA1_URL=http://admin:{SECRET}@10.9.100.6:3000", f"REPLICA1_PASSWORD={SECRET}",
+            "CRON=*/5 * * * *",
+        )
+        host.container(
+            "bunkerweb", "bunkerity/bunkerweb@sha256:2026",
+            "API_WHITELIST_IP=10.50.100.5", f"DATABASE_URI=mariadb+pymysql://bw:{SECRET}@db/bw",
+        )
+        host.container("zabbix", "zabbix/zabbix-agent2:alpine-latest", f"ZBX_PSK={SECRET}")
+
+        out = host.run()
+
+        assert SECRET not in out
+        assert "admin" not in out
+        reading = parse_service_clusters(out)
+        assert [m["address"] for m in reading["adguardhome_sync"][0]["members"]] == [
+            "10.9.100.5", "10.9.100.6",
+        ]
+        assert reading["bunkerweb"][0]["cluster"] == "10.50.100.5"
+        assert reading["patroni"] == []
+
+    def test_a_host_without_docker(self, host):
+        reading = parse_service_clusters(host.run(docker=False))
+
+        assert (reading["adguardhome_sync"], reading["bunkerweb"]) == ([], [])
+
+
+class TestTheMixin:
+    class _Driver(ServiceClustersMixin):
+        def __init__(self, answers: dict) -> None:
+            self.answers = answers
+            self.calls: list = []
+
+        def _run_service_clusters_command(self, command: str, *, privileged: bool) -> str:
+            self.calls.append(privileged)
+            return self.answers[privileged]
+
+    def test_root_first(self):
+        driver = self._Driver({True: _wire(), False: "unused"})
+
+        assert driver.get_service_clusters()["patroni"] == []
+        assert driver.calls == [True]
+
+    def test_without_root_when_root_brought_no_report(self):
+        driver = self._Driver({True: "sudo: a password is required", False: _wire()})
+
+        assert driver.get_service_clusters()["bunkerweb"] == []
+        assert driver.calls == [True, False]
+
+    def test_the_container_engine_binary_is_the_drivers(self):
+        class _Qnap(self._Driver):
+            def _container_engine_binary(self, engine: str) -> str:
+                return "/share/CACHEDEV1_DATA/.qpkg/container-station/bin/docker"
+
+        driver = _Qnap({True: _wire(), False: _wire()})
+        seen: list = []
+
+        def run(command: str, *, privileged: bool) -> str:
+            seen.append(command)
+            return _wire()
+
+        driver._run_service_clusters_command = run  # type: ignore[method-assign]
+        driver.get_service_clusters()
+
+        assert "container-station/bin/docker" in seen[0]
+
+    def test_the_default_command_asks_docker(self):
+        assert "docker" in SERVICE_CLUSTERS_COMMAND
+
+
+class TestTheSource:
+    """Where the evidence was read: the unit or container that carries it."""
+
+    def test_a_template_unit_is_named_after_its_instance(self):
+        reading = parse_service_clusters(
+            _wire(_patroni("active active", "scope pg", "name a", unit="patroni@16-main.service"))
+        )
+
+        assert reading["patroni"][0]["source"] == "patroni@16-main"
+
+    def test_a_container_by_its_name(self):
+        reading = parse_service_clusters(
+            _wire("", _container("adguardhome-sync", "bakito/adguardhome-sync", "ORIGIN_URL http://a:3000"),
+                  _container("bw", "bunkerity/bunkerweb", "API_WHITELIST_IP 10.0.0.5"))
+        )
+
+        assert reading["adguardhome_sync"][0]["source"] == "adguardhome-sync"
+        assert reading["bunkerweb"][0]["source"] == "bw"