"""Whether an image is what CIS asks of it (CIS Docker 1.6.0 §4, CIS Debian Linux 12). Trivy measures what it automates, ``image_tools/cis_os.sh`` what can be seen from inside the image, and ``cis.toml`` decides the rest. The gate is red for a failure nobody decided, an expired exception, a control neither measured nor decided, a decision for a control nothing knows, and a measurement not taken. """ from __future__ import annotations import datetime import json import subprocess import pytest from image_tools.cis import ( Control, Decision, check, from_os, from_trivy, judge, load_decisions, load_packages, merge, render, ) TODAY = datetime.date(2026, 10, 21) SUMMARY = { "ID": "docker-cis-1.6.0", "SummaryControls": [ {"ID": "4.1", "Name": "Ensure a user for the container has been created", "TotalFail": 1}, {"ID": "4.3", "Name": "Ensure unnecessary packages are not installed (Manual)"}, {"ID": "4.7", "Name": "Ensure update instructions are not used alone", "TotalFail": 0}, {"ID": "4.8", "Name": "Ensure setuid and setgid permissions are removed (Manual)"}, ], } OS = ( "docker:4.3\tPASS\tUnnecessary packages are not installed (packages.allow)\t\n" "docker:4.8\tFAIL\tsetuid and setgid permissions are removed\t/usr/bin/su /usr/bin/mount\n" "debian:7.2.5\tPASS\tNo duplicate UIDs exist\t\n" ) def _c(cid, result, detail=""): return Control(cid, cid, result, detail) class TestMeasurements: def test_trivy_without_a_count_did_not_measure(self): controls = {c.id: c for c in from_trivy(SUMMARY)} assert controls["docker:4.1"].result == "fail" assert controls["docker:4.3"].result == "unmeasured" assert controls["docker:4.7"].result == "pass" def test_the_os_checks_say_what_fails(self): controls = {c.id: c for c in from_os(OS)} assert controls["docker:4.8"].result == "fail" assert controls["docker:4.8"].detail == "/usr/bin/su /usr/bin/mount" assert controls["debian:7.2.5"].result == "pass" def test_a_line_the_os_checks_should_not_print_is_an_error(self): with pytest.raises(ValueError): from_os("sh: 1: stat: not found\n") def test_the_os_checks_settle_what_trivy_leaves_to_manual(self): controls = {c.id: c for c in merge(from_trivy(SUMMARY), from_os(OS))} assert controls["docker:4.3"].result == "pass" assert controls["docker:4.8"].result == "fail" assert controls["docker:4.1"].result == "fail" assert list(controls)[:4] == ["docker:4.1", "docker:4.3", "docker:4.7", "docker:4.8"] def test_a_failure_wins_over_a_pass(self): (control,) = merge([_c("docker:4.8", "fail")], [_c("docker:4.8", "pass")]) assert control.result == "fail" class TestJudge: def test_all_passing_is_green(self): assert judge([_c("docker:4.1", "pass")], {}, TODAY).ok def test_a_failure_nobody_decided_is_red(self): report = judge([_c("docker:4.1", "fail")], {}, TODAY) assert not report.ok assert report.controls[0].state == "fail" def test_an_exception_keeps_a_failure_green(self): decisions = {"docker:4.6": Decision("exception", "no service")} report = judge([_c("docker:4.6", "fail")], decisions, TODAY) assert report.ok assert report.controls[0].state == "excepted" def test_an_expired_exception_is_red(self): decisions = {"docker:4.6": Decision("exception", "for now", datetime.date(2026, 10, 1))} report = judge([_c("docker:4.6", "fail")], decisions, TODAY) assert not report.ok assert report.controls[0].state == "expired" def test_an_exception_expiring_today_still_holds(self): decisions = {"docker:4.6": Decision("exception", "for now", TODAY)} assert judge([_c("docker:4.6", "fail")], decisions, TODAY).ok def test_a_control_nothing_measures_must_be_decided(self): assert not judge([_c("docker:4.2", "unmeasured")], {}, TODAY).ok decisions = {"docker:4.2": Decision("manual", "pinned by digest")} report = judge([_c("docker:4.2", "unmeasured")], decisions, TODAY) assert report.ok assert report.controls[0].state == "manual" def test_manual_is_not_the_tools_word(self): """4.4 belongs to the matcher, whose policy is that new CVEs never block.""" decisions = {"docker:4.4": Decision("manual", "judged by the matcher")} assert judge([_c("docker:4.4", "fail")], decisions, TODAY).ok def test_an_exception_for_a_passing_control_is_named_stale(self): decisions = {"docker:4.6": Decision("exception", "no service")} report = judge([_c("docker:4.6", "pass")], decisions, TODAY) assert report.ok assert report.stale == ["docker:4.6"] def test_a_decision_for_a_control_nothing_knows_is_red(self): """A typo in cis.toml must not silently decide nothing.""" decisions = {"docker:4.66": Decision("exception", "typo")} report = judge([_c("docker:4.6", "pass")], decisions, TODAY) assert not report.ok assert report.unknown == ["docker:4.66"] def test_nothing_measured_is_not_green(self): assert not judge([], {}, TODAY).ok class TestDecisions: def test_both_kinds_with_their_reasons(self): decisions = load_decisions( """ [[manual]] id = "docker:4.2" reason = "pinned by digest" [[exception]] id = "docker:4.6" reason = "no service" expires = 2027-01-31 """ ) assert decisions["docker:4.2"] == Decision("manual", "pinned by digest") assert decisions["docker:4.6"].expires == datetime.date(2027, 1, 31) @pytest.mark.parametrize( "text", [ '[[manual]]\nid = "docker:4.2"\nreason = " "\n', '[[manual]]\nid = "docker:4.2"\nreason = "a"\n[[exception]]\nid = "docker:4.2"\nreason = "b"\n', '[[exception]]\nid = "docker:4.6"\nreason = "a"\nexpires = "soon"\n', ], ids=["no reason", "decided twice", "expires not a date"], ) def test_a_decision_without_substance_is_refused(self, text): with pytest.raises(ValueError): load_decisions(text) class TestCheck: def test_measured_and_judged(self): seams = {"trivy": lambda ref: SUMMARY, "os_checks": lambda ref, pkgs: OS} decisions = {"docker:4.8": Decision("exception", "not yet")} report = check("img", decisions, ["apt"], TODAY, **seams) assert not report.ok # 4.1 fails undecided assert {c.id: c.state for c in report.controls}["docker:4.8"] == "excepted" def test_the_packages_reach_the_os_checks(self): seen = [] def os_checks(ref, packages): seen.append(packages) return OS check("img", {}, ["apt", "bash"], TODAY, trivy=lambda r: SUMMARY, os_checks=os_checks) assert seen == [["apt", "bash"]] def test_a_measurement_not_taken_is_red(self): def trivy(ref): raise subprocess.CalledProcessError(1, "trivy") report = check("img", {}, [], TODAY, trivy=trivy, os_checks=lambda r, p: OS) assert not report.ok assert "CalledProcessError" in report.error def test_the_report_says_why(): decisions = { "docker:4.6": Decision("exception", "base-os runs no service"), "docker:4.2": Decision("manual", "pinned by digest"), } controls = [ _c("docker:4.1", "fail", "1 failed (Trivy)"), _c("docker:4.2", "unmeasured"), _c("docker:4.6", "fail", "1 failed (Trivy)"), ] text = render(judge(controls, decisions, TODAY), "base-os:ci") assert "**CIS: red**" in text and "`base-os:ci`" in text assert "| docker:4.1 | ❌ fail |" in text and "1 failed (Trivy)" in text assert "base-os runs no service" in text assert "pinned by digest" in text def test_the_report_is_json_for_the_attestation(): report = judge([_c("docker:4.1", "pass")], {}, TODAY) data = json.loads(report.to_json()) assert data["ok"] is True assert data["controls"][0] == { "id": "docker:4.1", "title": "docker:4.1", "result": "pass", "detail": "", "state": "pass", "reason": "", } def test_the_allow_list_is_names_without_comments_or_blank_lines(): text = "# what base-os may hold\napt\n\n bash \n# login.defs comes from login\nlogin\n" assert load_packages(text) == ["apt", "bash", "login"]