From 3b6c5f958550c6f3593fb00c2cd0f510fb0c0fe7 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 07:58:05 +0100 Subject: [PATCH 1/2] fix(apt_proxy): https sources take the http proxy too (NetOrk/netork#1811) apt's https method asks Acquire::https::Proxy and then Acquire::http::Proxy, for the host and then for all. An 00proxy with the http line alone therefore sent download.docker.com and every other https source to apt-cacher-ng, which answers 403 CONNECT denied without a PassThroughPattern, and the whole upgrade failed. - The report carries https_proxy, https_per_host and https_settings; https_proxy_for(report, host) is the proxy apt takes for one https host. - check_https_source() fetches one https source the way apt does. - configure_package_proxy gains an https step: one https source through the new proxy. Where the proxy refuses it, 00proxy also says Acquire::https::Proxy "DIRECT", later https lines to the same proxy are commented out, and the source is fetched direct. An HTTP error from the mirror counts as let through. - apt_fetch_command takes https URLs; comment_out_command takes a scheme. --- CHANGELOG.md | 17 ++ napalm_device_types/__init__.py | 2 + napalm_device_types/apt_proxy.py | 272 ++++++++++++++++++++++---- tests/test_apt_proxy.py | 319 +++++++++++++++++++++++++++++-- 4 files changed, 555 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8279e69..3312e3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,23 @@ recorded here; their history is the git log. link, owned by the login user. It reports every file's lines before and after, how many were removed and which files are root's again. Nothing goes to stdin. +### Fixed +- `AptProxyMixin` knows that apt's https sources take the http proxy too: apt's https + method asks `Acquire::https::Proxy` and then `Acquire::http::Proxy`, for the host and + then for all. An `00proxy` with the http line alone sent download.docker.com and + every other https source to apt-cacher-ng, which answers `403 CONNECT denied` without + a `PassThroughPattern`, and the whole upgrade failed (NetOrk/netork#1811). + - The report carries `https_proxy`, `https_per_host` and `https_settings`; + `https_proxy_for(report, host)` is the proxy apt takes for one https host. + - `check_https_source()` fetches one https source's `InRelease` the way apt does; + None where no https source goes through a proxy. + - `configure_package_proxy` gains an `https` step: one https source through the new + proxy. Where the proxy refuses it, `00proxy` also says `Acquire::https::Proxy + "DIRECT";`, later https lines to the same proxy are commented out, and the source + is fetched direct. An HTTP error from the mirror counts as let through. + - `apt_fetch_command` takes https URLs; `direct` sets the host's https setting too. + - `comment_out_command(..., scheme="https")`. + ## [4.22.0] - 2026-10-23 ### Added diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 35e95ed..a3d1e8f 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -72,6 +72,7 @@ from napalm_device_types.access_point import AccessPointDriver from napalm_device_types.apt_proxy import ( APT_PROXY_COMMAND, AptProxyMixin, + https_proxy_for, parse_apt_fetch, parse_apt_proxy, same_proxy_url, @@ -252,6 +253,7 @@ __all__ = [ "PackageProxyMixin", "APT_PROXY_COMMAND", "AptProxyMixin", + "https_proxy_for", "parse_apt_fetch", "parse_apt_proxy", "same_proxy_url", diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index 7cc7b6f..e8e933a 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -12,12 +12,18 @@ Debian installer writes its proxy into ``apt.conf``, Ubuntu's into (NetOrk/netork#1397). ``DIRECT`` for the proxy, or for one host (``Acquire::http::Proxy::``), means no proxy there. +**https** goes through the http proxy too. apt's https method asks +``Acquire::https::Proxy`` first and ``Acquire::http::Proxy`` after it, for one +host and then for all (NetOrk/netork#1811). An http proxy alone therefore takes +every https source as well, and a cache that does not tunnel them answers 403. + **Where it is set** is every line of apt's files that sets the http proxy, in the -order apt reads them. A line of the form ``Acquire::http::Proxy "…";`` can be -commented out; a setting inside a block (``Acquire::http { Proxy "…"; };``) is -named but left alone. apt skips a file in ``apt.conf.d`` whose name has other -characters than letters, digits, ``_``, ``-`` and ``.``, or an extension other -than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); so is it here. +order apt reads them; the https proxy's lines are named apart. A line of the form +``Acquire::http::Proxy "…";`` can be commented out; a setting inside a block +(``Acquire::http { Proxy "…"; };``) is named but left alone. apt skips a file in +``apt.conf.d`` whose name has other characters than letters, digits, ``_``, ``-`` +and ``.``, or an extension other than ``.conf`` (``00proxy.bak``, ``*.dpkg-old``); +so is it here. **Whether a proxy answers** is a fetch with apt's own downloader, ``apt-helper download-file``, which is on every apt host and needs no root: @@ -29,13 +35,15 @@ page is the right one, the caller says (``expect``). out every other line that sets a different one, reads the result back from ``apt-config``, and fetches the ``InRelease`` of one http source through it. The last step is what proves it: apt reached a mirror through the proxy it is now set -to. The source chosen is one apt does not fetch ``DIRECT``. +to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one +https source the same way; where the proxy refuses it, https goes ``DIRECT``. """ from __future__ import annotations import re import shlex +from collections.abc import Callable from typing import TYPE_CHECKING from urllib.parse import urlsplit @@ -47,7 +55,7 @@ from napalm_device_types.terminal import strip_terminal_codes class AptProxySettingDict(TypedDict): - """One line of apt's configuration that sets the http proxy (``AptProxyMixin``).""" + """One line of apt's configuration that sets a proxy (``AptProxyMixin``).""" file: str line: int # 1-based @@ -68,6 +76,14 @@ class AptProxyReportDict(TypedDict): auto_detect: str | None #: Every line that sets it, in the order apt reads them; the last one wins. settings: list[AptProxySettingDict] + #: The proxy apt takes for https: ``Acquire::https::Proxy``, else the http one; + #: None for none or ``DIRECT``. + https_proxy: str | None + #: The per-host proxies https takes: ``Acquire::https::Proxy::``, else + #: ``Acquire::http::Proxy::``. + https_per_host: dict[str, str] + #: Every line that sets ``Acquire::https::Proxy``, in the order apt reads them. + https_settings: list[AptProxySettingDict] #: The base URIs of the host's sources over http and over https. http_sources: list[str] https_sources: list[str] @@ -85,6 +101,15 @@ class ProxyFetchDict(TypedDict): error: str +class HttpsSourceFetchDict(ProxyFetchDict): + """One https source fetched the way apt fetches it (``check_https_source``).""" + + #: What was fetched: the source's ``InRelease``. + url: str + #: The proxy apt took for it. + proxy: str + + _BEGIN = "APROXY_BEGIN" _END = "APROXY_END" _APT_HELPER = "/usr/lib/apt/apt-helper" @@ -98,7 +123,7 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") #: them early. APT_PROXY_COMMAND = ( "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " - "echo '[dump]'; apt-config dump 2>/dev/null | grep -i '^Acquire::http::Proxy'; " + "echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; " 'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; ' "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " '[ -f "$f" ] && grep -Hin proxy "$f"; done; ' @@ -108,12 +133,15 @@ APT_PROXY_COMMAND = ( "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" ) -_DUMP = re.compile(r'^Acquire::http::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) +_DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) _GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") _FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) _NOT_HTTP_PROXY = re.compile( r"^\s*(//|#)|Acquire::(https|ftp|socks)\b|::Proxy::|Proxy-Auto-Detect", re.I ) +_FLAT_HTTPS = re.compile(r'^\s*Acquire::https::Proxy\s+"([^"]*)"\s*;', re.I) +#: A line that sets the https proxy inside a block, all on the line itself. +_HTTPS_BLOCK = re.compile(r'^\s*Acquire::https\s*\{[^}]*\bProxy\s+"([^"]*)"', re.I) _IN_BLOCK = re.compile(r'\bProxy\s+"([^"]*)"', re.I) #: The names apt reads in ``apt.conf.d``: no extension, or ``.conf``. _PART_NAME = re.compile(r"^(?:[A-Za-z0-9_\-]+|[A-Za-z0-9_.\-]+\.conf)$") @@ -138,13 +166,22 @@ def _sections(output: str) -> dict[str, list[str]]: return sections -def _setting(line: str, main: bool) -> AptProxySettingDict | None: +def _grep_line(line: str, main: bool) -> tuple[str, int, str] | None: + """File, line number and text of one line grep printed; None for a file apt skips.""" found = _GREP.match(line) if not found: return None path, number, text = found.groups() if not main and not _PART_NAME.match(path.rsplit("/", 1)[-1]): return None + return path, int(number), text + + +def _setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found flat = _FLAT.match(text) if flat: return {"file": path, "line": int(number), "value": flat.group(1), "editable": True} @@ -156,22 +193,72 @@ def _setting(line: str, main: bool) -> AptProxySettingDict | None: return {"file": path, "line": int(number), "value": nested.group(1), "editable": False} -def _dump(lines: list[str]) -> tuple[str | None, dict[str, str], str | None]: - """The proxy, the per-host proxies and the auto-detect program in ``apt-config dump``.""" - proxy: str | None = None - per_host: dict[str, str] = {} - auto_detect: str | None = None +def _https_setting(line: str, main: bool) -> AptProxySettingDict | None: + found = _grep_line(line, main) + if not found: + return None + path, number, text = found + flat = _FLAT_HTTPS.match(text) + if flat: + return {"file": path, "line": number, "value": flat.group(1), "editable": True} + block = _HTTPS_BLOCK.match(text) + if block: + return {"file": path, "line": number, "value": block.group(1), "editable": False} + return None + + +class _Dump(TypedDict): + proxy: str | None + per_host: dict[str, str] + auto_detect: str | None + #: ``Acquire::https::Proxy`` as written; None when unset or empty. + https: str | None + https_per_host: dict[str, str] + + +def _dump(lines: list[str]) -> _Dump: + """The proxies, the per-host proxies and the auto-detect program in ``apt-config dump``.""" + dump: _Dump = { + "proxy": None, + "per_host": {}, + "auto_detect": None, + "https": None, + "https_per_host": {}, + } for found in (_DUMP.match(line.strip()) for line in lines): if not found: continue - suffix, value = found.groups() - if not suffix: - proxy = None if value.lower() in _NO_PROXY else value + scheme, suffix, value = found.groups() + https = scheme.lower() == "https" + if not suffix and https: + dump["https"] = value or None + elif not suffix: + dump["proxy"] = None if value.lower() in _NO_PROXY else value elif suffix.lower() == "-auto-detect": - auto_detect = value or None - else: - per_host[suffix[2:]] = value - return proxy, per_host, auto_detect + if not https: + dump["auto_detect"] = value or None + elif value: + dump["https_per_host" if https else "per_host"][suffix[2:]] = value + return dump + + +def _https_proxy(dump: _Dump) -> str | None: + """apt's https method takes its own setting and falls back to the http one.""" + if dump["https"] is None: + return dump["proxy"] + return None if dump["https"].lower() in _NO_PROXY else dump["https"] + + +def https_proxy_for(report: AptProxyReportDict, host: str) -> str | None: + """The proxy apt takes for an https source on *host*; None for ``DIRECT`` or none. + + The host's own setting first, https before http, then the one for all hosts. + """ + for per_host in (report["https_per_host"], report["per_host"]): + value = next((v for h, v in per_host.items() if h.lower() == host.lower()), None) + if value: + return None if value.lower() in _NO_PROXY else value + return report["https_proxy"] def parse_apt_proxy(output: str) -> AptProxyReportDict: @@ -180,19 +267,21 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: :raises ValueError: if the output carried no intact report. """ sections = _sections(output) - proxy, per_host, auto_detect = _dump(sections.get("dump", [])) - settings = [ - s + dump = _dump(sections.get("dump", [])) + lines = [ + (main, line) for main, key in ((False, "parts"), (True, "main")) - for s in (_setting(line, main) for line in sections.get(key, [])) - if s + for line in sections.get(key, []) ] sources = [line.strip() for line in sections.get("sources", [])] return { - "proxy": proxy, - "per_host": per_host, - "auto_detect": auto_detect, - "settings": settings, + "proxy": dump["proxy"], + "per_host": dump["per_host"], + "auto_detect": dump["auto_detect"], + "settings": [s for s in (_setting(line, main) for main, line in lines) if s], + "https_proxy": _https_proxy(dump), + "https_per_host": {**dump["per_host"], **dump["https_per_host"]}, + "https_settings": [s for s in (_https_setting(line, main) for main, line in lines) if s], "http_sources": [s for s in sources if s.startswith("http://")], "https_sources": [s for s in sources if s.startswith("https://")], } @@ -225,14 +314,17 @@ def apt_fetch_command(url: str, *, direct: bool = False, expect: str = "") -> st *direct* goes past any proxy apt is set to, to *url*'s host itself; without it the fetch takes apt's own way. *expect* is text the page should contain. - :raises ValueError: if *url* is not an http URL a shell word can carry. + :raises ValueError: if *url* is not an http(s) URL a shell word can carry. """ - if not PROXY_URL.match(url) or not url.startswith("http://"): + if not PROXY_URL.match(url): raise ValueError(f"not an http URL: {url!r}") options = ["-o Acquire::Retries=0", "-o Acquire::http::Timeout=10"] host = urlsplit(url).hostname or "" if direct and host and ":" not in host: - # *host* comes from a URL PROXY_URL let through: no quote in it. + # *host* comes from a URL PROXY_URL let through: no quote in it. For https + # the host's https setting comes first, so both are set. + if url.startswith("https://"): + options.append(f"-o 'Acquire::https::Proxy::{host}=DIRECT'") options.append(f"-o 'Acquire::http::Proxy::{host}=DIRECT'") match = f"echo '[match]'; grep -c -F -- {shlex.quote(expect)} \"$t\"; " if expect else "" return ( @@ -272,13 +364,16 @@ def parse_apt_fetch(output: str) -> ProxyFetchDict: return {"reachable": answered, "expected": False if answered else None, "error": error} -#: sed's edit of one line: comment it out, if it still sets the http proxy. -_COMMENT_OUT = "-e '{line}{{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}}'" +#: sed's edit of one line: comment it out, if it still sets the proxy of *scheme*. +_COMMENT_OUT = "-e '{line}{{/^[[:space:]]*Acquire::{scheme}::Proxy[[:space:]]/Is|^|// |}}'" +_SCHEMES = ("http", "https") -def comment_out_command(path: str, lines: list[int]) -> str: - """The root command that comments out *lines* of *path* where they set the http proxy.""" - edits = " ".join(_COMMENT_OUT.format(line=int(n)) for n in lines) +def comment_out_command(path: str, lines: list[int], scheme: str = "http") -> str: + """The root command that comments out *lines* of *path* where they set *scheme*'s proxy.""" + if scheme not in _SCHEMES: + raise ValueError(f"not a scheme apt proxies: {scheme!r}") + edits = " ".join(_COMMENT_OUT.format(line=int(n), scheme=scheme) for n in lines) return f"sed -i {edits} {shlex.quote(path)}" @@ -333,6 +428,8 @@ class AptProxyMixin(PackageProxyMixin): * per_host ({host: proxy or "DIRECT"}) * auto_detect (string or None): ``Acquire::http::Proxy-Auto-Detect`` * settings (list of {file, line, value, editable}), in apt's reading order + * https_proxy (string or None): the proxy https takes, the http one if unset + * https_per_host ({host: proxy or "DIRECT"}), https_settings (as settings) * http_sources, https_sources (lists of the sources' base URIs) :raises ValueError: if the host's output carried no intact report. @@ -360,8 +457,30 @@ class AptProxyMixin(PackageProxyMixin): self._run_apt_command(apt_fetch_command(url, direct=True, expect=expect)) ) + def check_https_source(self) -> HttpsSourceFetchDict | None: + """Fetch one https source's ``InRelease`` the way apt does, through its proxy. + + None when no https source goes through a proxy: there are none, or https + goes ``DIRECT``. + + :raises ValueError: if the host's output carried no intact report. + :raises NotImplementedError: on a host without apt. + """ + if not self._is_apt_host(): + raise NotImplementedError("not an apt host") + report = self._report() + found = _https_source(report, lambda proxy: proxy is not None) + if found is None: + return None + target, proxy = found + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + return {**fetched, "url": target, "proxy": proxy} + def configure_package_proxy(self, url: str) -> FixResultDict: - """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`.""" + """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`. + + https goes through *url* too, unless *url* refuses it: then ``DIRECT``. + """ if not url: return run_fix_steps([("configure", self._remove_package_proxy)]) return run_fix_steps( @@ -370,6 +489,7 @@ class AptProxyMixin(PackageProxyMixin): ("conflicts", lambda: self._comment_out_other_proxies(url)), ("verify", lambda: self._verify_package_proxy(url)), ("fetch", lambda: self._fetch_through_package_proxy(url)), + ("https", lambda: self._https_through_package_proxy(url)), ] ) @@ -383,12 +503,15 @@ class AptProxyMixin(PackageProxyMixin): f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" ) - def _write_package_proxy(self, url: str) -> FixStepOutcomeDict: + def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() if not PROXY_URL.match(url): raise ValueError(f"not a proxy URL: {url!r}") - return self._write_apt_file(self.APT_PROXY_FILE, f'Acquire::http::Proxy "{url}";\n') + content = f'Acquire::http::Proxy "{url}";\n' + if https_direct: + content += 'Acquire::https::Proxy "DIRECT";\n' + return self._write_apt_file(self.APT_PROXY_FILE, content) def _report(self) -> AptProxyReportDict: # The steps after "configure" run only once it passed, on an apt host. @@ -442,16 +565,81 @@ class AptProxyMixin(PackageProxyMixin): return {"ok": True, "output": f"fetched {target} through {url}"} return {"ok": False, "output": f"{target}: {fetched['error']}"} + def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict: + """An https source through *url*; where *url* refuses it, https goes ``DIRECT``.""" + found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url)) + if found is None: + return {"ok": True, "output": f"no https source goes through {url}; not tried"} + target = found[0] + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + # An HTTP error came from the mirror, through the tunnel: the cache let it by. + if fetched["reachable"]: + return {"ok": True, "output": f"fetched {target} through {url}"} + refused = f"{url} refused https ({fetched['error']})" + outcome = self._https_direct(url) + if not outcome["ok"]: + return {"ok": False, "output": f"{refused}; {outcome['output']}"} + direct = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if not direct["reachable"]: + return {"ok": False, "output": f"{refused}; direct too: {target}: {direct['error']}"} + named = f"; {outcome['output']}" if outcome["output"] else "" + return { + "ok": True, + "output": f"{refused}{named}; https sources go direct now: fetched {target}", + } + + def _https_direct(self, url: str) -> FixStepOutcomeDict: + """Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*.""" + written = self._write_package_proxy(url, https_direct=True) + if not written["ok"]: + return written + later = [ + s + for s in self._report()["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + for path, numbers in _editable_lines(later).items(): + outcome = self._run_apt_root_command( + comment_out_command(path, numbers, scheme="https"), f"commented out in {path}" + ) + if not outcome["ok"]: + return outcome + report = self._report() + still = [ + f"{s['file']}:{s['line']}" + for s in report["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + if same_proxy_url(report["https_proxy"], url): + where = f", set in {', '.join(still)}" if still else "" + return {"ok": False, "output": f"https still goes through {url}{where}"} + return {"ok": True, "output": "; ".join(map(_described, later))} + + +def _https_source( + report: AptProxyReportDict, through: Callable[[str | None], bool] +) -> tuple[str, str] | None: + """The ``InRelease`` of the first https source whose proxy *through* accepts, and that proxy.""" + for source in report["https_sources"]: + if "/dists/" not in source: + continue + proxy = https_proxy_for(report, urlsplit(source).hostname or "") + if through(proxy): + return source.rstrip("/") + "/InRelease", proxy or "" + return None + __all__ = [ "APT_PROXY_COMMAND", "AptProxyMixin", "AptProxyReportDict", "AptProxySettingDict", + "HttpsSourceFetchDict", "ProxyFetchDict", "PROXY_URL", "apt_fetch_command", "comment_out_command", + "https_proxy_for", "parse_apt_fetch", "parse_apt_proxy", "same_proxy_url", diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py index 3071f2d..b886211 100644 --- a/tests/test_apt_proxy.py +++ b/tests/test_apt_proxy.py @@ -20,6 +20,7 @@ from napalm_device_types.apt_proxy import ( AptProxyMixin, apt_fetch_command, comment_out_command, + https_proxy_for, parse_apt_fetch, parse_apt_proxy, same_proxy_url, @@ -141,7 +142,7 @@ class TestWhereItIsSet: class TestSources: - def test_https_sources_bypass_an_http_proxy(self): + def test_sources_by_scheme(self): report = parse_apt_proxy( _report( sources=[ @@ -155,6 +156,102 @@ class TestSources: assert report["https_sources"] == ["https://download.docker.com/linux/ubuntu/dists/noble/"] +class TestHttpsProxy: + """apt's https method asks ``Acquire::https::Proxy`` and then ``Acquire::http::Proxy``: + an http proxy alone takes the https sources too (NetOrk/netork#1811). Measured with + apt 2.8.3's apt-helper against a dead port.""" + + def test_the_http_proxy_alone_takes_https_too(self): + report = parse_apt_proxy(_report(dump=['Acquire::http::Proxy "http://c:3142";'])) + assert report["https_proxy"] == "http://c:3142" + + def test_an_https_proxy_of_its_own_wins(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy "http://other:3128";', + ] + ) + ) + + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", "http://other:3128") + + @pytest.mark.parametrize("value", ["DIRECT", "direct"]) + def test_direct_https_is_none(self, value): + report = parse_apt_proxy( + _report( + dump=['Acquire::http::Proxy "http://c:3142";', f'Acquire::https::Proxy "{value}";'] + ) + ) + assert (report["proxy"], report["https_proxy"]) == ("http://c:3142", None) + + def test_an_empty_https_proxy_is_not_set(self): + """apt takes an empty value for unset and goes on to the http one.""" + report = parse_apt_proxy( + _report(dump=['Acquire::http::Proxy "http://c:3142";', 'Acquire::https::Proxy "";']) + ) + assert report["https_proxy"] == "http://c:3142" + + def test_per_host_https_then_http(self): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy::a.example "DIRECT";', + 'Acquire::http::Proxy::b.example "DIRECT";', + 'Acquire::https::Proxy::b.example "http://c:3142";', + ] + ) + ) + + assert report["https_per_host"] == {"a.example": "DIRECT", "b.example": "http://c:3142"} + assert report["per_host"] == {"a.example": "DIRECT", "b.example": "DIRECT"} + + def test_where_the_https_proxy_is_set(self): + report = parse_apt_proxy( + _report( + parts=[ + f'{_D}00proxy:1:Acquire::http::Proxy "http://c:3142";', + f'{_D}90curtin-aptproxy:2:Acquire::https::Proxy "http://c:3142";', + f'{_D}91x:1:Acquire::https {{ Proxy "http://n:1"; }};', + f'{_D}92x:1:// Acquire::https::Proxy "http://c:3142";', + f'{_D}93x:1:Acquire::https::Proxy::h.example "DIRECT";', + ] + ) + ) + + assert report["https_settings"] == [ + { + "file": f"{_D}90curtin-aptproxy", + "line": 2, + "value": "http://c:3142", + "editable": True, + }, + {"file": f"{_D}91x", "line": 1, "value": "http://n:1", "editable": False}, + ] + assert [s["file"] for s in report["settings"]] == [f"{_D}00proxy"] + + @pytest.mark.parametrize( + "host, proxy", + [ + ("deb.example", "http://c:3142"), + ("direct.example", None), + ("Other.Example", "http://o:1"), + ], + ) + def test_the_proxy_for_one_host(self, host, proxy): + report = parse_apt_proxy( + _report( + dump=[ + 'Acquire::http::Proxy "http://c:3142";', + 'Acquire::https::Proxy::direct.example "DIRECT";', + 'Acquire::http::Proxy::other.example "http://o:1";', + ] + ) + ) + assert https_proxy_for(report, host) == proxy + + class TestSameProxyUrl: @pytest.mark.parametrize( "a, b", @@ -199,6 +296,12 @@ E: Download Failed""" _FETCHED = """Get:1 http://172.22.8.50:3142/acng-report.html Fetched 14.0 kB in 0s (0 B/s)""" +#: apt-cacher-ng without a PassThroughPattern, as apt 2.8.3 tells it (NetOrk/netork#1811). +_CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/InRelease + Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] +E: Download Failed""" + def _fetch(output, rc, match=None): lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] @@ -243,6 +346,12 @@ class TestFetch: assert result["expected"] is False assert result["error"] == "503 Host not found [IP: 172.22.8.50 3142]" + def test_a_refused_tunnel(self): + result = parse_apt_fetch(_fetch(_CONNECT_DENIED, 100)) + + assert result["reachable"] is False + assert "403 CONNECT denied" in result["error"] + def test_without_apt_helper_nothing_could_be_tried(self): with pytest.raises(ValueError, match="apt-helper"): parse_apt_fetch(_fetch("sh: 1: /usr/lib/apt/apt-helper: not found", 127)) @@ -265,6 +374,21 @@ class TestFetchCommand: command = apt_fetch_command("http://archive.ubuntu.com/ubuntu/dists/noble/InRelease") assert "DIRECT" not in command + def test_an_https_source_through_apt_s_own_proxy(self): + command = apt_fetch_command( + "https://download.docker.com/linux/ubuntu/dists/noble/InRelease" + ) + assert "DIRECT" not in command + assert "download-file https://download.docker.com/" in command + + def test_past_the_proxy_to_an_https_host(self): + """For https the host's own https setting comes first; the http one alone + would lose to an https proxy for that host.""" + command = apt_fetch_command("https://h.example/x", direct=True) + + assert "'Acquire::https::Proxy::h.example=DIRECT'" in command + assert "'Acquire::http::Proxy::h.example=DIRECT'" in command + @pytest.mark.parametrize("url", ["http://c:3142/x'; rm -rf /", "ftp://c/x", "c:3142"]) def test_a_url_that_would_break_the_command_is_refused(self, url): with pytest.raises(ValueError): @@ -275,6 +399,7 @@ def test_the_commands_are_valid_posix_sh(): for command in ( APT_PROXY_COMMAND, apt_fetch_command("http://c:3142/x", direct=True, expect="y"), + apt_fetch_command("https://h.example/dists/x/InRelease"), ): subprocess.run(["sh", "-n", "-c", command], check=True) @@ -283,16 +408,28 @@ def test_the_commands_are_valid_posix_sh(): def test_the_report_command_on_this_host(): output = subprocess.run(["sh", "-c", APT_PROXY_COMMAND], capture_output=True, text=True).stdout report = parse_apt_proxy(output) - assert set(report) >= {"proxy", "settings", "https_sources"} + assert set(report) >= {"proxy", "https_proxy", "settings", "https_settings", "https_sources"} + + +_DOCKER = "https://download.docker.com/linux/ubuntu/dists/noble/" class _Host(AptProxyMixin): - """A Debian host: its apt files, and what apt-config would make of them.""" + """A Debian host: its apt files, and what apt-config would make of them. - def __init__(self, files=None, fetch_rc=0, apt=True): + *https_sources* are its sources over https. *tunnel* is whether the proxy lets + them through; *direct_rc* is how a fetch of one goes that takes no proxy. + """ + + def __init__( + self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0 + ): self.files = dict(files or {}) self.fetch_rc = fetch_rc self.apt = apt + self.https_sources = list(https_sources) + self.tunnel = tunnel + self.direct_rc = direct_rc self.root = [] self.fetched = [] @@ -309,26 +446,36 @@ class _Host(AptProxyMixin): out.append((path, n, text)) return out - def _effective(self): + def _effective(self, scheme="http"): value = None for _path, _n, text in self._settings(): - m = re.match(r'\s*Acquire::http::Proxy\s+"([^"]*)"', text, re.I) + m = re.match(rf'\s*Acquire::{scheme}::Proxy\s+"([^"]*)"', text, re.I) if m: value = m.group(1) return value def _run_apt_command(self, command): if command == APT_PROXY_COMMAND: - value = self._effective() + value, https = self._effective(), self._effective("https") + dump = [f'Acquire::http::Proxy "{value}";'] if value else [] + dump += [f'Acquire::https::Proxy "{https}";'] if https else [] return _report( - dump=[f'Acquire::http::Proxy "{value}";'] if value else [], + dump=dump, parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], main=[f"{p}:{n}:{t}" for p, n, t in self._settings() if not p.startswith(_D)], - sources=["http://deb.debian.org/debian/dists/trixie/"], + sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources], ) self.fetched.append(command) + if re.search(r"download-file '?https://", command): + return self._fetch_https(command) return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) + def _fetch_https(self, command): + https = self._effective("https") or self._effective() + if "DIRECT" in command or not https or https.upper() == "DIRECT": + return _fetch(_FETCHED if self.direct_rc == 0 else _UNRESOLVED, self.direct_rc) + return _fetch(_FETCHED, 0) if self.tunnel else _fetch(_CONNECT_DENIED, 100) + def _write_apt_file(self, path, content): self.files[path] = content return {"ok": True, "output": f"wrote {path}"} @@ -346,6 +493,41 @@ class _Host(AptProxyMixin): return {"ok": True, "output": done} +class TestCheckHttpsSource: + def test_one_source_the_way_apt_fetches_it(self): + host = _Host( + {f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.check_https_source() + + assert result["url"] == f"{_DOCKER}InRelease" + assert result["proxy"] == "http://c:3142" + assert result["reachable"] is False and "403 CONNECT denied" in result["error"] + assert "DIRECT" not in host.fetched[0] + + def test_none_without_an_https_source(self): + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + assert host.check_https_source() is None + + def test_none_where_https_goes_direct(self): + host = _Host( + { + f"{_D}00proxy": ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + }, + https_sources=[_DOCKER], + ) + assert host.check_https_source() is None + + def test_not_on_a_host_without_apt(self): + with pytest.raises(NotImplementedError): + _Host(apt=False).check_https_source() + + class TestTheMixin: def test_it_is_a_package_proxy(self): assert issubclass(AptProxyMixin, PackageProxyMixin) @@ -371,6 +553,10 @@ class TestTheMixin: assert "Proxy::c=DIRECT" in host.fetched[0] +def _step(result, name): + return next(s for s in result["steps"] if s["step"] == name) + + class TestConfigurePackageProxy: def test_a_fresh_host(self): host = _Host() @@ -378,7 +564,14 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") assert result["success"] is True - assert [s["step"] for s in result["steps"]] == ["configure", "conflicts", "verify", "fetch"] + assert [s["step"] for s in result["steps"]] == [ + "configure", + "conflicts", + "verify", + "fetch", + "https", + ] + assert "no https source" in result["steps"][-1]["output"] assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] @@ -410,7 +603,7 @@ class TestConfigurePackageProxy: def test_a_setting_it_cannot_comment_out_fails_verification(self): host = _Host({"/etc/apt/apt.conf": 'Acquire::http { Proxy "http://older:8080"; };\n'}) - host._effective = lambda: "http://older:8080" + host._effective = lambda scheme="http": "http://older:8080" result = host.configure_package_proxy("http://c:3142") @@ -428,6 +621,100 @@ class TestConfigurePackageProxy: assert result["steps"][-1]["step"] == "fetch" assert "Connection refused" in result["steps"][-1]["output"] + def test_an_https_source_the_cache_tunnels_stays_with_it(self): + host = _Host(https_sources=[_DOCKER]) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' + assert result["steps"][-1]["output"] == ( + f"fetched {_DOCKER}InRelease through http://c:3142" + ) + + def test_an_https_source_the_cache_refuses_goes_direct(self): + """NetOrk/netork#1811: apt-cacher-ng without a PassThroughPattern answers + 403 to every https source apt sends it.""" + host = _Host(https_sources=[_DOCKER], tunnel=False) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == ( + 'Acquire::http::Proxy "http://c:3142";\nAcquire::https::Proxy "DIRECT";\n' + ) + output = result["steps"][-1]["output"] + assert "http://c:3142 refused https" in output and "403 CONNECT denied" in output + assert f"https sources go direct now: fetched {_DOCKER}InRelease" in output + + def test_a_later_https_line_to_the_cache_is_commented_out(self): + """Ubuntu's installer writes the https proxy too; it would win over 00proxy.""" + curtin = f"{_D}90curtin-aptproxy" + host = _Host( + { + curtin: ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Acquire::https::Proxy "http://c:3142";\n' + ) + }, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[curtin] == ( + 'Acquire::http::Proxy "http://c:3142";\n// Acquire::https::Proxy "http://c:3142";\n' + ) + assert f"{curtin}:2 (http://c:3142) commented out" in result["steps"][-1]["output"] + + def test_an_https_proxy_of_its_own_is_not_the_fix_s(self): + host = _Host( + {f"{_D}01https": 'Acquire::https::Proxy "http://corp:3128";\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "no https source goes through http://c:3142" in result["steps"][-1]["output"] + assert f"{_D}00proxy" in host.files and "https" not in host.files[f"{_D}00proxy"] + + def test_an_http_error_through_the_tunnel_is_a_tunnel(self): + """The mirror answered, so the cache let the source through.""" + host = _Host(https_sources=[_DOCKER]) + host._fetch_https = lambda command: _fetch(_HTTP_ERROR, 100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert "https" not in host.files[f"{_D}00proxy"] + + def test_refused_and_no_way_direct_fails(self): + host = _Host(https_sources=[_DOCKER], tunnel=False, direct_rc=100) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "Could not resolve" in result["steps"][-1]["output"] + + def test_a_later_https_line_it_cannot_comment_out_fails(self): + host = _Host( + {"/etc/apt/apt.conf": 'Acquire::https { Proxy "http://c:3142"; };\n'}, + https_sources=[_DOCKER], + tunnel=False, + ) + host._effective = lambda scheme="http": "http://c:3142" + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1]["step"] == "https" + assert "/etc/apt/apt.conf:1" in result["steps"][-1]["output"] + def test_without_an_http_source_the_fetch_says_so(self): host = _Host() real = host._run_apt_command @@ -440,7 +727,7 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") assert result["success"] is True - assert "no http source" in result["steps"][-1]["output"] + assert "no http source" in _step(result, "fetch")["output"] def test_a_direct_source_is_not_proof(self): host = _Host() @@ -451,7 +738,7 @@ class TestConfigurePackageProxy: result = host.configure_package_proxy("http://c:3142") - assert "no http source" in result["steps"][-1]["output"] + assert "no http source" in _step(result, "fetch")["output"] @pytest.mark.parametrize("url", ['http://x"; Acquire::x "y', "ftp://c", "c:3142"]) def test_a_url_that_would_break_the_file_is_refused(self, url): @@ -531,3 +818,9 @@ def test_the_comment_out_command(): "sed -i -e '2{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' " "-e '4{/^[[:space:]]*Acquire::http::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" ) + + +def test_the_comment_out_command_for_https(): + assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == ( + "sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" + ) -- 2.54.0 From c71ddd7fe6c7f48de4df7517e41a3a09f9a57656 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 1 Nov 2026 08:13:20 +0100 Subject: [PATCH 2/2] refactor(apt_proxy): smaller parts, under the standard's complexity limit The quality ratchet counted _dump, parse_apt_proxy and _https_direct over complexity 10 (NetOrk/netork#1811). Same behaviour, same tests. --- napalm_device_types/apt_proxy.py | 73 ++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index e8e933a..daa70c3 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -226,22 +226,31 @@ def _dump(lines: list[str]) -> _Dump: "https_per_host": {}, } for found in (_DUMP.match(line.strip()) for line in lines): - if not found: - continue - scheme, suffix, value = found.groups() - https = scheme.lower() == "https" - if not suffix and https: - dump["https"] = value or None - elif not suffix: - dump["proxy"] = None if value.lower() in _NO_PROXY else value - elif suffix.lower() == "-auto-detect": - if not https: - dump["auto_detect"] = value or None - elif value: - dump["https_per_host" if https else "per_host"][suffix[2:]] = value + if found: + _take(dump, *found.groups()) return dump +def _take(dump: _Dump, scheme: str, suffix: str | None, value: str) -> None: + """One line of the dump into *dump*; a later one of the same key wins, as in apt.""" + https = scheme.lower() == "https" + if not suffix: + _take_for_all(dump, https, value) + elif suffix.lower() == "-auto-detect": + if not https: + dump["auto_detect"] = value or None + elif value: + dump["https_per_host" if https else "per_host"][suffix[2:]] = value + + +def _take_for_all(dump: _Dump, https: bool, value: str) -> None: + """The proxy for every host: https keeps an empty value as unset, apt's fallback.""" + if https: + dump["https"] = value or None + else: + dump["proxy"] = None if value.lower() in _NO_PROXY else value + + def _https_proxy(dump: _Dump) -> str | None: """apt's https method takes its own setting and falls back to the http one.""" if dump["https"] is None: @@ -278,15 +287,25 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: "proxy": dump["proxy"], "per_host": dump["per_host"], "auto_detect": dump["auto_detect"], - "settings": [s for s in (_setting(line, main) for main, line in lines) if s], + "settings": _settings(lines, _setting), "https_proxy": _https_proxy(dump), "https_per_host": {**dump["per_host"], **dump["https_per_host"]}, - "https_settings": [s for s in (_https_setting(line, main) for main, line in lines) if s], - "http_sources": [s for s in sources if s.startswith("http://")], - "https_sources": [s for s in sources if s.startswith("https://")], + "https_settings": _settings(lines, _https_setting), + "http_sources": _of_scheme(sources, "http"), + "https_sources": _of_scheme(sources, "https"), } +def _settings( + lines: list[tuple[bool, str]], parse: Callable[[str, bool], AptProxySettingDict | None] +) -> list[AptProxySettingDict]: + return [s for s in (parse(line, main) for main, line in lines) if s] + + +def _of_scheme(sources: list[str], scheme: str) -> list[str]: + return [s for s in sources if s.startswith(f"{scheme}://")] + + _DEFAULT_PORT = {"http": 80, "https": 443} @@ -593,11 +612,7 @@ class AptProxyMixin(PackageProxyMixin): written = self._write_package_proxy(url, https_direct=True) if not written["ok"]: return written - later = [ - s - for s in self._report()["https_settings"] - if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) - ] + later = self._later_https_lines(self._report(), url) for path, numbers in _editable_lines(later).items(): outcome = self._run_apt_root_command( comment_out_command(path, numbers, scheme="https"), f"commented out in {path}" @@ -605,16 +620,20 @@ class AptProxyMixin(PackageProxyMixin): if not outcome["ok"]: return outcome report = self._report() - still = [ - f"{s['file']}:{s['line']}" - for s in report["https_settings"] - if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) - ] if same_proxy_url(report["https_proxy"], url): + still = [f"{s['file']}:{s['line']}" for s in self._later_https_lines(report, url)] where = f", set in {', '.join(still)}" if still else "" return {"ok": False, "output": f"https still goes through {url}{where}"} return {"ok": True, "output": "; ".join(map(_described, later))} + def _later_https_lines(self, report: AptProxyReportDict, url: str) -> list[AptProxySettingDict]: + """The lines outside :attr:`APT_PROXY_FILE` that send https to *url*.""" + return [ + s + for s in report["https_settings"] + if s["file"] != self.APT_PROXY_FILE and same_proxy_url(s["value"], url) + ] + def _https_source( report: AptProxyReportDict, through: Callable[[str | None], bool] -- 2.54.0