feat: AptProxyMixin, what apt fetches through and pointing it at a proxy (NetOrk/netork#1397)

apt reads apt.conf.d in order and apt.conf last; the last value wins. The Debian installer writes its proxy into apt.conf, Ubuntu's into 90curtin-aptproxy: reading 00proxy alone missed both, and a fix that wrote 00proxy and read it back reported success while apt fetched elsewhere. - get_package_proxy_report(): apt-config dump's proxy, per-host overrides, auto-detect, every line of apt's files that sets it (in reading order, editable or inside a block), the http and https sources. - check_package_proxy(url, expect): a fetch with apt-helper, past any proxy; an HTTP error counts as an answer. - configure_package_proxy(url): write 00proxy, comment out other proxy lines, read apt's proxy back, fetch one InRelease through it. - same_proxy_url(): scheme, host and port, a trailing slash aside. A driver supplies _run_apt_command, _run_apt_root_command and _write_apt_file. Version 4.22.0.
2026-10-22 19:16:40 +02:00
1 parent ac8e7c4c11
commit d7ec558933
@@ -0,0 +1,475 @@
+"""apt's proxy: what apt fetches through, whether a proxy answers, pointing apt at one.
+
+apt reads every file in ``apt.conf.d`` in order and ``apt.conf`` last; the last
+value wins. The Debian installer writes its proxy into ``apt.conf``, Ubuntu's
+into ``apt.conf.d/90curtin-aptproxy``: reading one file was not enough
+(NetOrk/netork#1397). The fetch outputs are apt-helper's own, from a Mint 22.3
+host against netOrk's test server.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+
+import pytest
+
+from napalm_device_types.apt_proxy import (
+    APT_PROXY_COMMAND,
+    AptProxyMixin,
+    apt_fetch_command,
+    parse_apt_fetch,
+    parse_apt_proxy,
+    same_proxy_url,
+)
+from napalm_device_types.fixes import PackageProxyMixin
+
+
+def _report(dump=(), parts=(), main=(), sources=()):
+    lines = ["APROXY_BEGIN", "[dump]", *dump, "[parts]", *parts, "[main]", *main]
+    return "\n".join([*lines, "[sources]", *sources, "APROXY_END"])
+
+
+_D = "/etc/apt/apt.conf.d/"
+
+
+class TestEffectiveProxy:
+    def test_the_proxy_apt_fetches_through(self):
+        report = parse_apt_proxy(
+            _report(
+                dump=['Acquire::http::Proxy "http://c:3142";'],
+                parts=[f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";'],
+            )
+        )
+
+        assert report["proxy"] == "http://c:3142"
+        assert report["settings"] == [
+            {"file": f"{_D}00proxy", "line": 1, "value": "http://c:3142", "editable": True}
+        ]
+
+    def test_none_without_a_proxy(self):
+        report = parse_apt_proxy(_report())
+
+        assert report["proxy"] is None
+        assert report["settings"] == []
+        assert report["per_host"] == {}
+        assert report["auto_detect"] is None
+
+    @pytest.mark.parametrize("value", ["", "DIRECT", "direct"])
+    def test_an_empty_or_direct_proxy_is_none(self, value):
+        assert parse_apt_proxy(_report(dump=[f'Acquire::http::Proxy "{value}";']))["proxy"] is None
+
+    def test_keys_are_case_insensitive(self):
+        report = parse_apt_proxy(_report(dump=['acquire::HTTP::proxy "http://c:3142";']))
+        assert report["proxy"] == "http://c:3142"
+
+    def test_per_host_overrides_and_auto_detect(self):
+        report = parse_apt_proxy(
+            _report(
+                dump=[
+                    'Acquire::http::Proxy "http://c:3142";',
+                    'Acquire::http::Proxy::deb.debian.org "DIRECT";',
+                    'Acquire::http::Proxy::ppa.launchpad.net "http://other:3128";',
+                    'Acquire::http::Proxy-Auto-Detect "/usr/bin/auto-apt-proxy";',
+                ]
+            )
+        )
+
+        assert report["proxy"] == "http://c:3142"
+        assert report["per_host"] == {
+            "deb.debian.org": "DIRECT",
+            "ppa.launchpad.net": "http://other:3128",
+        }
+        assert report["auto_detect"] == "/usr/bin/auto-apt-proxy"
+
+    def test_no_intact_report_raises(self):
+        with pytest.raises(ValueError):
+            parse_apt_proxy("APROXY_BEGIN\n[dump]\n")
+
+
+class TestWhereItIsSet:
+    def test_every_setting_in_apt_s_reading_order(self):
+        report = parse_apt_proxy(
+            _report(
+                dump=['Acquire::http::Proxy "http://old:3128";'],
+                parts=[
+                    f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";',
+                    f'{_D}90curtin-aptproxy:1:Acquire::http::Proxy "http://old:3128";',
+                ],
+                main=['/etc/apt/apt.conf:4:  Acquire::http::Proxy "http://older:8080";'],
+            )
+        )
+
+        assert [(s["file"], s["line"], s["value"]) for s in report["settings"]] == [
+            (f"{_D}00proxy", 1, "http://c:3142"),
+            (f"{_D}90curtin-aptproxy", 1, "http://old:3128"),
+            ("/etc/apt/apt.conf", 4, "http://older:8080"),
+        ]
+
+    def test_a_nested_setting_is_named_but_not_editable(self):
+        report = parse_apt_proxy(
+            _report(parts=[f'{_D}01proxy:2:    Proxy "http://nested:3142";'])
+        )
+
+        assert report["settings"] == [
+            {"file": f"{_D}01proxy", "line": 2, "value": "http://nested:3142", "editable": False}
+        ]
+
+    @pytest.mark.parametrize(
+        "line",
+        [
+            '// Acquire::http::Proxy "http://c:3142";',
+            '# Acquire::http::Proxy "http://c:3142";',
+            'Acquire::https::Proxy "http://c:3142";',
+            'Acquire::ftp::Proxy "http://c:3142";',
+            'Acquire::http::Proxy::deb.debian.org "DIRECT";',
+            'Acquire::http::Proxy-Auto-Detect "/usr/bin/x";',
+            "Acquire::http {",
+        ],
+    )
+    def test_what_does_not_set_the_http_proxy_is_left_out(self, line):
+        assert parse_apt_proxy(_report(parts=[f"{_D}00x:1:{line}"]))["settings"] == []
+
+    @pytest.mark.parametrize("name", ["00proxy.bak", "00proxy.dpkg-old", "00proxy.save", "a b"])
+    def test_files_apt_does_not_read_are_left_out(self, name):
+        line = f'{_D}{name}:1:Acquire::http::Proxy "http://c:3142";'
+        assert parse_apt_proxy(_report(parts=[line]))["settings"] == []
+
+    def test_a_conf_extension_is_read(self):
+        line = f'{_D}01proxy.conf:1:Acquire::http::Proxy "http://c:3142";'
+        assert len(parse_apt_proxy(_report(parts=[line]))["settings"]) == 1
+
+
+class TestSources:
+    def test_https_sources_bypass_an_http_proxy(self):
+        report = parse_apt_proxy(
+            _report(
+                sources=[
+                    "http://archive.ubuntu.com/ubuntu/dists/noble/",
+                    "https://download.docker.com/linux/ubuntu/dists/noble/",
+                ]
+            )
+        )
+
+        assert report["http_sources"] == ["http://archive.ubuntu.com/ubuntu/dists/noble/"]
+        assert report["https_sources"] == ["https://download.docker.com/linux/ubuntu/dists/noble/"]
+
+
+class TestSameProxyUrl:
+    @pytest.mark.parametrize(
+        "a, b",
+        [
+            ("http://c:3142", "http://c:3142/"),
+            ("http://C:3142", "http://c:3142"),
+            ("http://c", "http://c:80"),
+        ],
+    )
+    def test_the_same(self, a, b):
+        assert same_proxy_url(a, b)
+
+    @pytest.mark.parametrize(
+        "a, b",
+        [
+            ("http://c:3142", "http://c:31420"),
+            ("http://c:3142", "http://c:3142.evil"),
+            ("http://c:3142", "https://c:3142"),
+            ("http://c:3142", ""),
+            ("http://c:3142", None),
+        ],
+    )
+    def test_not_the_same(self, a, b):
+        assert not same_proxy_url(a, b)
+
+
+_REFUSED = """Err:1 http://172.22.8.50:3143/acng-report.html
+  Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused)
+E: Failed to fetch http://172.22.8.50:3143/acng-report.html  Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused)
+E: Download Failed"""
+
+_HTTP_ERROR = """Err:1 http://172.22.8.50:3142/nothing-here.html
+  503  Host not found [IP: 172.22.8.50 3142]
+E: Failed to fetch http://172.22.8.50:3142/nothing-here.html  503  Host not found [IP: 172.22.8.50 3142]
+E: Download Failed"""
+
+_UNRESOLVED = """Err:1 http://no-such-host.invalid:3142/acng-report.html
+  Could not resolve 'no-such-host.invalid'
+E: Failed to fetch http://no-such-host.invalid:3142/acng-report.html  Could not resolve 'no-such-host.invalid'
+E: Download Failed"""
+
+_FETCHED = """Get:1 http://172.22.8.50:3142/acng-report.html
+Fetched 14.0 kB in 0s (0 B/s)"""
+
+
+def _fetch(output, rc, match=None):
+    lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)]
+    if match is not None:
+        lines += ["[match]", str(match)]
+    return "\n".join([*lines, "APROXY_END"])
+
+
+class TestFetch:
+    def test_fetched_and_expected(self):
+        assert parse_apt_fetch(_fetch(_FETCHED, 0, match=4)) == {
+            "reachable": True,
+            "expected": True,
+            "error": "",
+        }
+
+    def test_fetched_but_not_the_expected_page(self):
+        assert parse_apt_fetch(_fetch(_FETCHED, 0, match=0))["expected"] is False
+
+    def test_without_an_expectation_expected_is_none(self):
+        assert parse_apt_fetch(_fetch(_FETCHED, 0))["expected"] is None
+
+    def test_refused(self):
+        result = parse_apt_fetch(_fetch(_REFUSED, 100))
+
+        assert result["reachable"] is False
+        assert result["error"] == (
+            "Could not connect to 172.22.8.50:3143 (172.22.8.50). - connect (111: Connection refused)"
+        )
+
+    def test_unresolved(self):
+        result = parse_apt_fetch(_fetch(_UNRESOLVED, 100))
+        assert (result["reachable"], result["error"]) == (
+            False,
+            "Could not resolve 'no-such-host.invalid'",
+        )
+
+    def test_an_http_error_was_an_answer(self):
+        result = parse_apt_fetch(_fetch(_HTTP_ERROR, 100))
+
+        assert result["reachable"] is True
+        assert result["expected"] is False
+        assert result["error"] == "503  Host not found [IP: 172.22.8.50 3142]"
+
+    def test_without_apt_helper_nothing_could_be_tried(self):
+        with pytest.raises(ValueError, match="apt-helper"):
+            parse_apt_fetch(_fetch("sh: 1: /usr/lib/apt/apt-helper: not found", 127))
+
+    def test_no_intact_report_raises(self):
+        with pytest.raises(ValueError):
+            parse_apt_fetch("APROXY_BEGIN\n[fetch]\n")
+
+
+class TestFetchCommand:
+    def test_past_the_proxy_to_the_url_itself(self):
+        command = apt_fetch_command("http://c:3142/acng-report.html", direct=True, expect="Cache NG")
+
+        assert "'Acquire::http::Proxy::c=DIRECT'" in command
+        assert "grep -c -F -- 'Cache NG'" in command
+
+    def test_through_apt_s_own_proxy(self):
+        command = apt_fetch_command("http://archive.ubuntu.com/ubuntu/dists/noble/InRelease")
+        assert "DIRECT" not in command
+
+    @pytest.mark.parametrize("url", ["http://c:3142/x'; rm -rf /", "ftp://c/x", "c:3142"])
+    def test_a_url_that_would_break_the_command_is_refused(self, url):
+        with pytest.raises(ValueError):
+            apt_fetch_command(url)
+
+
+def test_the_commands_are_valid_posix_sh():
+    for command in (APT_PROXY_COMMAND, apt_fetch_command("http://c:3142/x", direct=True, expect="y")):
+        subprocess.run(["sh", "-n", "-c", command], check=True)
+
+
+@pytest.mark.skipif(not os.path.exists("/usr/bin/apt-config"), reason="needs an apt host")
+def test_the_report_command_on_this_host():
+    output = subprocess.run(["sh", "-c", APT_PROXY_COMMAND], capture_output=True, text=True).stdout
+    report = parse_apt_proxy(output)
+    assert set(report) >= {"proxy", "settings", "https_sources"}
+
+
+class _Host(AptProxyMixin):
+    """A Debian host: its apt files, and what apt-config would make of them."""
+
+    def __init__(self, files=None, fetch_rc=0, apt=True):
+        self.files = dict(files or {})
+        self.fetch_rc = fetch_rc
+        self.apt = apt
+        self.root = []
+        self.fetched = []
+
+    def _is_apt_host(self):
+        return self.apt
+
+    def _settings(self):
+        out = []
+        for path in sorted(p for p in self.files if p.startswith(_D)) + [
+            p for p in self.files if p == "/etc/apt/apt.conf"
+        ]:
+            for n, text in enumerate(self.files[path].splitlines(), 1):
+                if "proxy" in text.lower():
+                    out.append((path, n, text))
+        return out
+
+    def _effective(self):
+        value = None
+        for _path, _n, text in self._settings():
+            m = re.match(r'\s*Acquire::http::Proxy\s+"([^"]*)"', text, re.I)
+            if m:
+                value = m.group(1)
+        return value
+
+    def _run_apt_command(self, command):
+        if command == APT_PROXY_COMMAND:
+            value = self._effective()
+            return _report(
+                dump=[f'Acquire::http::Proxy "{value}";'] if value else [],
+                parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)],
+                main=[f"{p}:{n}:{t}" for p, n, t in self._settings() if not p.startswith(_D)],
+                sources=["http://deb.debian.org/debian/dists/trixie/"],
+            )
+        self.fetched.append(command)
+        return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1)
+
+    def _write_apt_file(self, path, content):
+        self.files[path] = content
+        return {"ok": True, "output": f"wrote {path}"}
+
+    def _run_apt_root_command(self, command, done):
+        self.root.append(command)
+        path = command.rsplit(" ", 1)[-1].strip("'")
+        for m in re.finditer(r"(\d+)\{/[^}]*\}", command):
+            n = int(m.group(1))
+            lines = self.files[path].splitlines()
+            lines[n - 1] = "// " + lines[n - 1]
+            self.files[path] = "\n".join(lines) + "\n"
+        if command.startswith("rm -f "):
+            self.files.pop(command.split()[-1].strip("'"), None)
+        return {"ok": True, "output": done}
+
+
+class TestTheMixin:
+    def test_it_is_a_package_proxy(self):
+        assert issubclass(AptProxyMixin, PackageProxyMixin)
+
+    def test_the_fact(self):
+        host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'})
+
+        assert host.get_package_proxy() == "http://c:3142"
+        assert host.get_package_proxy_report()["settings"][0]["file"] == f"{_D}00proxy"
+
+    def test_not_an_apt_host(self):
+        host = _Host(apt=False)
+
+        assert host.get_package_proxy() is None
+        assert host.get_package_proxy_report() is None
+
+    def test_check_fetches_the_page_past_any_proxy(self):
+        host = _Host()
+
+        result = host.check_package_proxy("http://c:3142/acng-report.html", expect="Cache")
+
+        assert result == {"reachable": True, "expected": True, "error": ""}
+        assert "Proxy::c=DIRECT" in host.fetched[0]
+
+
+class TestConfigurePackageProxy:
+    def test_a_fresh_host(self):
+        host = _Host()
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is True
+        assert [s["step"] for s in result["steps"]] == ["configure", "conflicts", "verify", "fetch"]
+        assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n'
+        assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0]
+
+    def test_a_later_file_is_commented_out_and_named(self):
+        host = _Host(
+            {
+                f"{_D}90curtin-aptproxy": 'Acquire::http::Proxy "http://old:3128";\n',
+                "/etc/apt/apt.conf": 'APT::Foo "1";\nAcquire::http::Proxy "http://older:8080";\n',
+            }
+        )
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is True
+        assert host.files[f"{_D}90curtin-aptproxy"].startswith("// Acquire::http::Proxy")
+        assert host.files["/etc/apt/apt.conf"] == (
+            'APT::Foo "1";\n// Acquire::http::Proxy "http://older:8080";\n'
+        )
+        conflicts = result["steps"][1]["output"]
+        assert f"{_D}90curtin-aptproxy:1 (http://old:3128)" in conflicts
+        assert "/etc/apt/apt.conf:2 (http://older:8080)" in conflicts
+
+    def test_the_same_proxy_elsewhere_stays(self):
+        host = _Host({f"{_D}01proxy": 'Acquire::http::Proxy "http://c:3142/";\n'})
+
+        host.configure_package_proxy("http://c:3142")
+
+        assert host.root == []
+
+    def test_a_setting_it_cannot_comment_out_fails_verification(self):
+        host = _Host({"/etc/apt/apt.conf": 'Acquire::http { Proxy "http://older:8080"; };\n'})
+        host._effective = lambda: "http://older:8080"
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is False
+        assert result["steps"][-1]["step"] == "verify"
+        assert "http://older:8080" in result["steps"][-1]["output"]
+        assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"]
+
+    def test_a_dead_proxy_fails_the_fetch(self):
+        host = _Host(fetch_rc=100)
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is False
+        assert result["steps"][-1]["step"] == "fetch"
+        assert "Connection refused" in result["steps"][-1]["output"]
+
+    def test_without_an_http_source_the_fetch_says_so(self):
+        host = _Host()
+        real = host._run_apt_command
+
+        def no_sources(command):
+            return real(command).replace("http://deb.debian.org/debian/dists/trixie/\n", "")
+
+        host._run_apt_command = no_sources
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is True
+        assert "no http source" in result["steps"][-1]["output"]
+
+    def test_a_direct_source_is_not_proof(self):
+        host = _Host()
+        real = host._run_apt_command
+        host._run_apt_command = lambda command: real(command).replace(
+            "[parts]", 'Acquire::http::Proxy::deb.debian.org "DIRECT";\n[parts]'
+        )
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert "no http source" in result["steps"][-1]["output"]
+
+    @pytest.mark.parametrize("url", ['http://x"; Acquire::x "y', "ftp://c", "c:3142"])
+    def test_a_url_that_would_break_the_file_is_refused(self, url):
+        host = _Host()
+
+        result = host.configure_package_proxy(url)
+
+        assert result["success"] is False
+        assert host.files == {}
+
+    def test_an_empty_url_removes_the_file(self):
+        host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'})
+
+        result = host.configure_package_proxy("")
+
+        assert result["success"] is True
+        assert host.files == {}
+
+    def test_only_an_apt_host_takes_one(self):
+        host = _Host(apt=False)
+
+        result = host.configure_package_proxy("http://c:3142")
+
+        assert result["success"] is False
+        assert host.files == {}