From cb0f1cc05c9e46737ba17535cc7401f9765dee4f Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 15 Nov 2026 20:07:27 +0100 Subject: [PATCH] feat(apt): send https sources through the cache in the form it stores (NetOrk/netork#2555) apt-cacher-ng tunnels a source fetched over https and stores nothing of it. It fetches http://HTTPS//// over https itself, with the certificate verified, and stores it, where port 443 is allowed (AllowUserPorts=80 443). - apt_sources: cacheable_https_hosts(report), and the root commands that rewrite the https sources of .list and deb822 .sources files to that form and back. A host apt has credentials for in auth.conf stays on https; the rewrite names it in the proxy file as Package-Proxy::Not-Cached, which the report reads. - configure_package_proxy(url): a step "cache" before "https". It rewrites the sources, fetches one of them through url, and turns them all back to https where url cannot. The "fetch" step proves the proxy with a plain http source. - configure_package_proxy(""): the sources go back to https before the proxy file goes; without the cache apt would look for a host called HTTPS. --- CHANGELOG.md | 17 ++ napalm_device_types/apt_proxy.py | 123 +++++++++-- napalm_device_types/apt_sources.py | 172 ++++++++++++++++ tests/test_apt_proxy.py | 196 +++++++++++++++++- tests/test_apt_sources.py | 318 +++++++++++++++++++++++++++++ 5 files changed, 808 insertions(+), 18 deletions(-) create mode 100644 napalm_device_types/apt_sources.py create mode 100644 tests/test_apt_sources.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c10b3..6ed0b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ recorded here; their history is the git log. ## [Unreleased] +### Added +- `apt_sources`: apt's https sources through the package cache, in the form apt-cacher-ng + fetches over https itself and stores (`http://HTTPS////…`, with `AllowUserPorts=80 + 443` on the cache). `cacheable_https_hosts(report)` names the hosts this applies to, and + `rewrite_sources_command` / `revert_sources_command` are the root commands, for `.list` + and deb822 `.sources` files. A host apt keeps credentials for in `auth.conf` stays on + https, because apt finds them by the host's name and the cache is shared. The rewrite + names such a host in the proxy file as `Package-Proxy::Not-Cached` (NetOrk/netork#2555). +- `get_package_proxy_report()` reports those hosts as `not_cached`. + +### Changed +- `configure_package_proxy(url)` has a step `cache` before `https`. It rewrites the https + sources it can, fetches one of them through *url*, and turns them all back to https + where *url* cannot. The `fetch` step proves the proxy with a plain http source only. +- `configure_package_proxy("")` turns the sources back to https (step `sources`) before it + removes the proxy file. Without the cache, apt would look for a host called `HTTPS`. + ## [4.39.0] - 2026-11-14 ### Added diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index daa70c3..9be8052 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -35,8 +35,12 @@ page is the right one, the caller says (``expect``). out every other line that sets a different one, reads the result back from ``apt-config``, and fetches the ``InRelease`` of one http source through it. The last step is what proves it: apt reached a mirror through the proxy it is now set -to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one -https source the same way; where the proxy refuses it, https goes ``DIRECT``. +to. The source chosen is one apt does not fetch ``DIRECT``. Then the https sources +go through the proxy in the form it stores (:mod:`~napalm_device_types.apt_sources`), +and one of them is fetched; where the proxy cannot, they go back to https. Last it +fetches one https source left the same way; where the proxy refuses it, https goes +``DIRECT``. Removing the proxy turns the sources back to https first: without the +proxy apt cannot fetch the stored form. """ from __future__ import annotations @@ -49,6 +53,14 @@ from urllib.parse import urlsplit from typing_extensions import TypedDict +from napalm_device_types.apt_sources import ( + NOT_CACHED_KEY, + VIA_CACHE, + cacheable_https_hosts, + is_host_name, + revert_sources_command, + rewrite_sources_command, +) from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps from napalm_device_types.models import FixResultDict, FixStepOutcomeDict from napalm_device_types.terminal import strip_terminal_codes @@ -87,6 +99,9 @@ class AptProxyReportDict(TypedDict): #: The base URIs of the host's sources over http and over https. http_sources: list[str] https_sources: list[str] + #: The hosts the cache rewrite left on https, for their credentials + #: (:data:`~napalm_device_types.apt_sources.NOT_CACHED_KEY`). + not_cached: list[str] class ProxyFetchDict(TypedDict): @@ -123,7 +138,8 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") #: them early. APT_PROXY_COMMAND = ( "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " - "echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; " + "echo '[dump]'; apt-config dump 2>/dev/null " + "| grep -i -E '^(Acquire::https?::Proxy|Package-Proxy::)'; " 'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; ' "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " '[ -f "$f" ] && grep -Hin proxy "$f"; done; ' @@ -133,6 +149,7 @@ APT_PROXY_COMMAND = ( "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" ) +_NOT_CACHED = re.compile(rf'^{re.escape(NOT_CACHED_KEY)}::\s+"([^"]+)"', re.I) _DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) _GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") _FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) @@ -293,6 +310,7 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: "https_settings": _settings(lines, _https_setting), "http_sources": _of_scheme(sources, "http"), "https_sources": _of_scheme(sources, "https"), + "not_cached": [m.group(1) for m in map(_NOT_CACHED.match, sections.get("dump", [])) if m], } @@ -498,16 +516,20 @@ class AptProxyMixin(PackageProxyMixin): def configure_package_proxy(self, url: str) -> FixResultDict: """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`. - https goes through *url* too, unless *url* refuses it: then ``DIRECT``. + The https sources go through *url* in the form it stores, where it can; the + rest goes through *url* too, unless *url* refuses it: then ``DIRECT``. """ if not url: - return run_fix_steps([("configure", self._remove_package_proxy)]) + return run_fix_steps( + [("sources", self._revert_sources), ("configure", self._remove_package_proxy)] + ) return run_fix_steps( [ ("configure", lambda: self._write_package_proxy(url)), ("conflicts", lambda: self._comment_out_other_proxies(url)), ("verify", lambda: self._verify_package_proxy(url)), ("fetch", lambda: self._fetch_through_package_proxy(url)), + ("cache", lambda: self._cache_https_sources(url)), ("https", lambda: self._https_through_package_proxy(url)), ] ) @@ -515,6 +537,11 @@ class AptProxyMixin(PackageProxyMixin): def _not_apt(self) -> FixStepOutcomeDict: return {"ok": False, "output": "not an apt host"} + def _revert_sources(self) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + return self._run_apt_root_command(revert_sources_command(), "sources back on https") + def _remove_package_proxy(self) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() @@ -522,7 +549,9 @@ class AptProxyMixin(PackageProxyMixin): f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" ) - def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict: + def _write_package_proxy( + self, url: str, https_direct: bool = False, not_cached: list[str] | None = None + ) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() if not PROXY_URL.match(url): @@ -530,6 +559,10 @@ class AptProxyMixin(PackageProxyMixin): content = f'Acquire::http::Proxy "{url}";\n' if https_direct: content += 'Acquire::https::Proxy "DIRECT";\n' + # Host names the report read from this file; one a line could not carry is left out. + content += "".join( + f'{NOT_CACHED_KEY}:: "{host}";\n' for host in not_cached or [] if is_host_name(host) + ) return self._write_apt_file(self.APT_PROXY_FILE, content) def _report(self) -> AptProxyReportDict: @@ -566,16 +599,7 @@ class AptProxyMixin(PackageProxyMixin): } def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict: - report = self._report() - direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} - source = next( - ( - s - for s in report["http_sources"] - if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct - ), - None, - ) + source = _plain_http_source(self._report()) if source is None: return {"ok": True, "output": "no http source to fetch through the proxy; not tried"} target = source.rstrip("/") + "/InRelease" @@ -584,6 +608,49 @@ class AptProxyMixin(PackageProxyMixin): return {"ok": True, "output": f"fetched {target} through {url}"} return {"ok": False, "output": f"{target}: {fetched['error']}"} + def _cache_https_sources(self, url: str) -> FixStepOutcomeDict: + """The https sources through *url* in the form it stores; back to https where it cannot.""" + hosts = cacheable_https_hosts(self._report()) + if not hosts: + return {"ok": True, "output": "no https source to cache"} + rewritten = self._run_apt_root_command( + rewrite_sources_command(hosts, self.APT_PROXY_FILE), "sources rewritten" + ) + if not rewritten["ok"]: + return rewritten + return self._fetch_cached(url, hosts, self._report()) + + def _fetch_cached( + self, url: str, hosts: list[str], report: AptProxyReportDict + ) -> FixStepOutcomeDict: + """One rewritten source through *url*: it stays where *url* fetched it.""" + left = sorted(set(hosts) & set(report["not_cached"])) + named = f"; left on https, apt has credentials for them: {', '.join(left)}" if left else "" + target = next( + (s.rstrip("/") + "/InRelease" for s in report["http_sources"] if _cached(s)), None + ) + if target is None: + return {"ok": True, "output": f"no https source to cache{named}"} + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if fetched["reachable"] and not fetched["error"]: + cached = ", ".join(h for h in hosts if h not in left) + return {"ok": True, "output": f"https sources cached through {url}: {cached}{named}"} + return self._back_to_https(url, f"{target}: {fetched['error']}", named) + + def _back_to_https(self, url: str, why: str, named: str) -> FixStepOutcomeDict: + """*url* did not fetch the stored form: every source goes back to https.""" + back = self._run_apt_root_command(revert_sources_command(), "sources back on https") + if not back["ok"]: + return { + "ok": False, + "output": f"{why}; turning the sources back to https failed: {back['output']}", + } + return { + "ok": True, + "output": f"{url} does not fetch https sources itself ({why}); " + f"they stay on https, not cached{named}", + } + def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict: """An https source through *url*; where *url* refuses it, https goes ``DIRECT``.""" found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url)) @@ -609,7 +676,9 @@ class AptProxyMixin(PackageProxyMixin): def _https_direct(self, url: str) -> FixStepOutcomeDict: """Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*.""" - written = self._write_package_proxy(url, https_direct=True) + written = self._write_package_proxy( + url, https_direct=True, not_cached=self._report()["not_cached"] + ) if not written["ok"]: return written later = self._later_https_lines(self._report(), url) @@ -635,6 +704,26 @@ class AptProxyMixin(PackageProxyMixin): ] +def _plain_http_source(report: AptProxyReportDict) -> str | None: + """A suite's http source the proxy takes, not in the cache's https form: its proof.""" + direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} + return next( + ( + s + for s in report["http_sources"] + if "/dists/" in s + and not s.startswith(VIA_CACHE) + and (urlsplit(s).hostname or "").lower() not in direct + ), + None, + ) + + +def _cached(source: str) -> bool: + """A source of a suite in the form the cache stores.""" + return source.startswith(VIA_CACHE) and "/dists/" in source + + def _https_source( report: AptProxyReportDict, through: Callable[[str | None], bool] ) -> tuple[str, str] | None: diff --git a/napalm_device_types/apt_sources.py b/napalm_device_types/apt_sources.py new file mode 100644 index 0000000..2f052ce --- /dev/null +++ b/napalm_device_types/apt_sources.py @@ -0,0 +1,172 @@ +"""apt's https sources through the package cache, so that the cache stores them. + +A source fetched over https goes through apt-cacher-ng's tunnel, if at all: the +TLS runs end to end and nothing of it is stored. The cache fetches +``http://HTTPS////`` from ``https:///`` itself, with the +certificate verified, and stores it -- where it allows port 443 +(``AllowUserPorts=80 443``) (NetOrk/netork#2555). apt fetches that form through +its http proxy, so it works only while the cache is that proxy: without one apt +looks for a host called ``HTTPS``. + +**Which hosts** (:func:`cacheable_https_hosts`): the hosts of the https sources, +on port 443, while apt has an http proxy for every host and none of its own for +that one. Not a host with a login in the URL, nor one whose credentials apt keeps +in ``auth.conf``: apt finds those by the host's name, which the rewrite changes, +and paid content does not belong in a cache other hosts share. Only root reads +``auth.conf``, so the rewrite judges that, leaves such a host on https and names +it in the proxy file as ``Package-Proxy::Not-Cached``, which apt ignores and +``apt-config dump`` shows to the report. + +**The rewrite** (:func:`rewrite_sources_command`) turns ``https://`` into +the cache form on the lines that name sources: ``deb`` and ``deb-src`` in a +``.list`` file, ``URIs:`` in a deb822 ``.sources`` file. A file without the host +is not touched. :func:`revert_sources_command` turns every source back. +""" + +from __future__ import annotations + +import re +import shlex +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +if TYPE_CHECKING: # pragma: no cover - the report's type, without an import cycle + from napalm_device_types.apt_proxy import AptProxyReportDict + +#: The prefix apt-cacher-ng fetches over https. +VIA_CACHE = "http://HTTPS///" + +#: The key in the proxy file that names the hosts left on https for their credentials. +NOT_CACHED_KEY = "Package-Proxy::Not-Cached" + +_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?" +_HOST = re.compile(rf"^{_LABEL}(?:\.{_LABEL})*$") +_NO_PROXY = frozenset({"", "direct"}) + +#: Where apt's sources and credentials are, as shell variables; apt's defaults +#: when apt-config cannot say. +_LOCATE = ( + 'eval "$({apt_config} shell sl Dir::Etc::sourcelist/f sp Dir::Etc::sourceparts/d ' + 'nr Dir::Etc::netrc/f np Dir::Etc::netrcparts/d 2>/dev/null)"; ' + 'sl="${{sl:-/etc/apt/sources.list}}"; sp="${{sp:-/etc/apt/sources.list.d/}}"; ' + 'nr="${{nr:-/etc/apt/auth.conf}}"; np="${{np:-/etc/apt/auth.conf.d/}}"; ' +) + +#: The lines that name sources: ``deb``/``deb-src`` and deb822's ``URIs:``. +_SOURCE_LINES = "/^[[:space:]]*(deb|deb-src|[Uu][Rr][Ii][Ss]:)/" + +#: The hosts apt has credentials for: each word after ``machine``, without its +#: scheme, port and path, in lower case. +_CREDENTIAL_HOSTS = ( + 'creds=$(cat "$nr" "$np"* 2>/dev/null | tr -s " \\t\\r" "\\n" ' + "| awk 'p {{ print; p = 0; next }} $0 == \"machine\" {{ p = 1 }}' " + "| sed -e 's|^[A-Za-z+]*://||' -e 's|[:/].*||' | tr A-Z a-z); " +) + +_REWRITE = ( + "( " + _LOCATE + _CREDENTIAL_HOSTS + "for h in {hosts}; do " + 'l=$(printf "%s" "$h" | tr A-Z a-z); ' + 'if printf "%s\\n" "$creds" | grep -qxF -- "$l"; then ' + 'grep -qF -- "\\"$h\\"" {proxy_file} 2>/dev/null || ' + 'printf \'{key}:: "%s";\\n\' "$h" >> {proxy_file} || exit 1; continue; fi; ' + "e=$(printf '%s' \"$h\" | sed 's/[.]/[.]/g'); " + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do ' + '[ -f "$f" ] && grep -qF -- "https://$h" "$f" || continue; ' + 'sed -i -E "{lines} s#([[:space:]:])https://$e([/[:space:]]|\\$)#\\1{via}$h\\2#g" "$f" ' + "|| exit 1; done; done )" +) + +_REVERT = ( + "( " + _LOCATE + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do ' + '[ -f "$f" ] && grep -qF -- "{via}" "$f" || continue; ' + "sed -i -E '{lines} s#([[:space:]:]){via}#\\1https://#g' \"$f\" || exit 1; done )" +) + + +def cache_form(url: str) -> str: + """*url*, an https URL, in the form apt-cacher-ng fetches over https and stores. + + :raises ValueError: if *url* is not an https URL. + """ + if not url.startswith("https://"): + raise ValueError(f"not an https URL: {url!r}") + return VIA_CACHE + url[len("https://") :] + + +def _per_host_direct(report: AptProxyReportDict) -> set[str]: + hosts = {**report["per_host"], **report["https_per_host"]} + return {h.lower() for h, v in hosts.items() if v.lower() in _NO_PROXY} + + +def _host_of(source: str) -> str | None: + """The host of an https source the cache can fetch: port 443, no login in the URL.""" + try: + parts = urlsplit(source) + port = parts.port + except ValueError: + return None + if parts.username is not None or port not in (None, 443) or not parts.hostname: + return None + return parts.hostname + + +def cacheable_https_hosts(report: AptProxyReportDict) -> list[str]: + """The hosts of the https sources the rewrite would send through the cache, sorted. + + None while apt has no http proxy for every host: the cache form goes through it. + Not a host apt fetches ``DIRECT``, nor one the rewrite left on https for its + credentials (``not_cached``), nor one on another port or with a login in the URL. + """ + if not report["proxy"]: + return [] + skip = _per_host_direct(report) | {h.lower() for h in report.get("not_cached", [])} + hosts = {_host_of(s) for s in report["https_sources"]} + return sorted(h for h in hosts if h and h.lower() not in skip) + + +def is_host_name(host: str) -> bool: + """Whether *host* is a DNS name: letters, digits and hyphens between dots.""" + return bool(_HOST.match(host)) + + +def _checked(hosts: list[str]) -> str: + for host in hosts: + if not is_host_name(host): + raise ValueError(f"not a host name: {host!r}") + return " ".join(hosts) + + +def rewrite_sources_command( + hosts: list[str], proxy_file: str, apt_config: str = "apt-config" +) -> str: + """The root command that sends the https sources of *hosts* through the cache. + + A host apt has credentials for stays on https and is named in *proxy_file*. + *apt_config* is how apt-config is called; a test points apt's paths elsewhere. + + :raises ValueError: if one of *hosts* is not a host name. + """ + return _REWRITE.format( + apt_config=apt_config, + hosts=_checked(hosts), + proxy_file=shlex.quote(proxy_file), + key=NOT_CACHED_KEY, + lines=_SOURCE_LINES, + via=VIA_CACHE, + ) + + +def revert_sources_command(apt_config: str = "apt-config") -> str: + """The root command that turns every source in the cache form back to https.""" + return _REVERT.format(apt_config=apt_config, lines=_SOURCE_LINES, via=VIA_CACHE) + + +__all__ = [ + "NOT_CACHED_KEY", + "VIA_CACHE", + "cache_form", + "cacheable_https_hosts", + "is_host_name", + "revert_sources_command", + "rewrite_sources_command", +] diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py index b886211..d8f8da8 100644 --- a/tests/test_apt_proxy.py +++ b/tests/test_apt_proxy.py @@ -25,6 +25,7 @@ from napalm_device_types.apt_proxy import ( parse_apt_proxy, same_proxy_url, ) +from napalm_device_types.apt_sources import NOT_CACHED_KEY, VIA_CACHE from napalm_device_types.fixes import PackageProxyMixin @@ -302,6 +303,13 @@ _CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/ E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] E: Download Failed""" +#: A cache that does not fetch the cache form itself: ``AllowUserPorts`` is 80 alone +#: (apt-cacher-ng 3.7.5, NetOrk/netork#2555). +_PROHIBITED_PORT = """Err:1 http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease + 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143] +E: Failed to fetch http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143] +E: Download Failed""" + def _fetch(output, rc, match=None): lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] @@ -422,9 +430,21 @@ class _Host(AptProxyMixin): """ def __init__( - self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0 + self, + files=None, + fetch_rc=0, + apt=True, + https_sources=(), + tunnel=True, + direct_rc=0, + cache_https=False, + credentials=(), ): self.files = dict(files or {}) + #: Whether the proxy fetches the cache form itself (AllowUserPorts 443), and + #: the hosts apt keeps credentials for. + self.cache_https = cache_https + self.credentials = set(credentials) self.fetch_rc = fetch_rc self.apt = apt self.https_sources = list(https_sources) @@ -459,6 +479,12 @@ class _Host(AptProxyMixin): value, https = self._effective(), self._effective("https") dump = [f'Acquire::http::Proxy "{value}";'] if value else [] dump += [f'Acquire::https::Proxy "{https}";'] if https else [] + dump += [ + line + for path, text in self.files.items() + for line in text.splitlines() + if line.startswith(NOT_CACHED_KEY) + ] return _report( dump=dump, parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], @@ -466,6 +492,8 @@ class _Host(AptProxyMixin): sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources], ) self.fetched.append(command) + if re.search(r"download-file '?http://HTTPS///", command): + return _fetch(_FETCHED, 0) if self.cache_https else _fetch(_PROHIBITED_PORT, 100) if re.search(r"download-file '?https://", command): return self._fetch_https(command) return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) @@ -482,6 +510,11 @@ class _Host(AptProxyMixin): def _run_apt_root_command(self, command, done): self.root.append(command) + if NOT_CACHED_KEY in command: + return self._rewrite(re.search(r"for h in ([^;]+); do", command).group(1).split(), done) + if "#\\1https://#g" in command: + self.https_sources = [s.replace(VIA_CACHE, "https://") for s in self.https_sources] + return {"ok": True, "output": done} path = command.rsplit(" ", 1)[-1].strip("'") for m in re.finditer(r"(\d+)\{/[^}]*\}", command): n = int(m.group(1)) @@ -492,6 +525,19 @@ class _Host(AptProxyMixin): self.files.pop(command.split()[-1].strip("'"), None) return {"ok": True, "output": done} + def _rewrite(self, hosts, done): + for host in hosts: + if host in self.credentials: + proxy = self.files.get(f"{_D}00proxy", "") + self.files[f"{_D}00proxy"] = proxy + f'{NOT_CACHED_KEY}:: "{host}";\n' + continue + prefix = f"https://{host}/" + self.https_sources = [ + VIA_CACHE + s[len("https://") :] if s.startswith(prefix) else s + for s in self.https_sources + ] + return {"ok": True, "output": done} + class TestCheckHttpsSource: def test_one_source_the_way_apt_fetches_it(self): @@ -569,8 +615,10 @@ class TestConfigurePackageProxy: "conflicts", "verify", "fetch", + "cache", "https", ] + assert "no https source" in _step(result, "cache")["output"] assert "no https source" in result["steps"][-1]["output"] assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] @@ -766,6 +814,144 @@ class TestConfigurePackageProxy: assert host.files == {} +class TestCacheHttpsSources: + """NetOrk/netork#2555: the cache stores an https source only in its own form.""" + + def test_a_cache_that_fetches_https_gets_them(self): + host = _Host(https_sources=[_DOCKER], cache_https=True) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.https_sources == [f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"] + assert _step(result, "cache")["output"] == ( + "https sources cached through http://c:3142: download.docker.com" + ) + assert any(VIA_CACHE in c for c in host.fetched) + assert "no https source goes through" in _step(result, "https")["output"] + + def test_a_cache_that_does_not_turns_them_back(self): + host = _Host(https_sources=[_DOCKER]) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.https_sources == [_DOCKER] + output = _step(result, "cache")["output"] + assert "http://c:3142 does not fetch https sources itself" in output + assert "prohibited port (see AllowUserPorts)" in output + assert "they stay on https, not cached" in output + assert _step(result, "https")["output"] == ( + f"fetched {_DOCKER}InRelease through http://c:3142" + ) + + def test_a_host_with_credentials_stays_on_https_and_is_named(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host( + https_sources=[pve, _DOCKER], + cache_https=True, + credentials={"enterprise.proxmox.com"}, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert pve in host.https_sources + output = _step(result, "cache")["output"] + assert "cached through http://c:3142: download.docker.com" in output + assert "left on https, apt has credentials for them: enterprise.proxmox.com" in output + assert host.get_package_proxy_report()["not_cached"] == ["enterprise.proxmox.com"] + + def test_only_hosts_with_credentials(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host(https_sources=[pve], cache_https=True, credentials={"enterprise.proxmox.com"}) + + result = host.configure_package_proxy("http://c:3142") + + assert _step(result, "cache")["output"] == ( + "no https source to cache; " + "left on https, apt has credentials for them: enterprise.proxmox.com" + ) + + def test_the_https_direct_fallback_keeps_the_hosts_left_on_https(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host(https_sources=[pve], tunnel=False, credentials={"enterprise.proxmox.com"}) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Acquire::https::Proxy "DIRECT";\n' + f'{NOT_CACHED_KEY}:: "enterprise.proxmox.com";\n' + ) + + def test_a_rewrite_that_fails_stops_the_fix(self): + host = _Host(https_sources=[_DOCKER], cache_https=True) + real = host._run_apt_root_command + + def denied(command, done): + if NOT_CACHED_KEY in command: + return {"ok": False, "output": "sed: denied"} + return real(command, done) + + host._run_apt_root_command = denied + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1] == {"step": "cache", "ok": False, "output": "sed: denied"} + + def test_a_turn_back_that_fails_says_so(self): + host = _Host(https_sources=[_DOCKER]) + real = host._run_apt_root_command + + def denied(command, done): + if "#\\1https://#g" in command: + return {"ok": False, "output": "sed: denied"} + return real(command, done) + + host._run_apt_root_command = denied + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + output = result["steps"][-1]["output"] + assert "turning the sources back to https failed: sed: denied" in output + + def test_the_proof_of_the_proxy_is_a_plain_http_source(self): + """A source already in the cache form says nothing about plain http.""" + host = _Host(https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"]) + + host.configure_package_proxy("http://c:3142") + + fetch = next(c for c in host.fetched if "InRelease" in c) + assert "deb.debian.org" in fetch + + def test_removing_the_proxy_turns_the_sources_back_first(self): + host = _Host( + {f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}, + https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"], + ) + + result = host.configure_package_proxy("") + + assert result["success"] is True + assert [s["step"] for s in result["steps"]] == ["sources", "configure"] + assert host.https_sources == [_DOCKER] + assert host.files == {} + + def test_a_turn_back_that_fails_keeps_the_proxy(self): + """Without the cache apt looks for a host called HTTPS: the proxy stays.""" + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + host._run_apt_root_command = lambda command, done: {"ok": False, "output": "denied"} + + result = host.configure_package_proxy("") + + assert result["success"] is False + assert f"{_D}00proxy" in host.files + + class TestWhatTheParserSkips: def test_a_line_that_is_no_grep_output(self): assert ( @@ -824,3 +1010,11 @@ def test_the_comment_out_command_for_https(): assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == ( "sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" ) + + +def test_a_line_left_on_https_that_is_no_host_name_is_not_written_back(): + host = _Host() + host._write_package_proxy("http://c:3142", not_cached=["ok.example", 'x"; Foo "1']) + assert host.files[f"{_D}00proxy"] == ( + f'Acquire::http::Proxy "http://c:3142";\n{NOT_CACHED_KEY}:: "ok.example";\n' + ) diff --git a/tests/test_apt_sources.py b/tests/test_apt_sources.py new file mode 100644 index 0000000..5db305c --- /dev/null +++ b/tests/test_apt_sources.py @@ -0,0 +1,318 @@ +"""https sources through the cache: which can be, and the root commands that turn them. + +apt-cacher-ng fetches ``http://HTTPS////`` from ``https:///`` +and stores it, where it allows port 443 (``AllowUserPorts=80 443``); a source +fetched over https itself goes through the cache's tunnel and is never stored +(NetOrk/netork#2555). The commands run here for real, on files in a temporary +directory that ``apt-config -o`` points apt's own settings at. +""" + +from __future__ import annotations + +import shutil +import subprocess + +import pytest + +from napalm_device_types.apt_proxy import parse_apt_proxy +from napalm_device_types.apt_sources import ( + VIA_CACHE, + cache_form, + cacheable_https_hosts, + is_host_name, + revert_sources_command, + rewrite_sources_command, +) + + +def _report(**over): + report = { + "proxy": "http://c:3142", + "per_host": {}, + "auto_detect": None, + "settings": [], + "https_proxy": "http://c:3142", + "https_per_host": {}, + "https_settings": [], + "http_sources": ["http://deb.debian.org/debian/dists/trixie/"], + "https_sources": [], + "not_cached": [], + } + report.update(over) + return report + + +class TestCacheForm: + def test_an_https_url(self): + assert cache_form("https://download.docker.com/linux/ubuntu/dists/noble/") == ( + "http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/" + ) + + def test_the_prefix(self): + assert VIA_CACHE == "http://HTTPS///" + + def test_not_an_https_url(self): + with pytest.raises(ValueError): + cache_form("http://deb.debian.org/debian/") + + +class TestCacheableHttpsHosts: + def test_each_host_once_sorted(self): + report = _report( + https_sources=[ + "https://packages.wazuh.com/4.x/apt/dists/stable/", + "https://download.docker.com/linux/ubuntu/dists/noble/", + "https://download.docker.com/linux/ubuntu/dists/noble-test/", + ] + ) + assert cacheable_https_hosts(report) == ["download.docker.com", "packages.wazuh.com"] + + def test_none_without_an_http_proxy(self): + # The cache form goes through the http proxy: without one it goes nowhere. + report = _report(proxy=None, https_sources=["https://h.example/dists/x/"]) + assert cacheable_https_hosts(report) == [] + + def test_a_host_apt_fetches_direct_stays(self): + report = _report( + per_host={"a.example": "DIRECT"}, + https_per_host={"B.example": "direct"}, + https_sources=[ + "https://a.example/dists/x/", + "https://b.example/dists/x/", + "https://c.example/dists/x/", + ], + ) + assert cacheable_https_hosts(report) == ["c.example"] + + def test_a_host_with_credentials_stays(self): + report = _report( + https_sources=["https://enterprise.proxmox.com/debian/pve/dists/trixie/"], + not_cached=["enterprise.proxmox.com"], + ) + assert cacheable_https_hosts(report) == [] + + def test_another_port_or_a_login_in_the_url_stays(self): + # The cache allows port 443 alone, and a login in the URL is a credential too. + report = _report( + https_sources=[ + "https://h.example:8443/dists/x/", + "https://u:p@i.example/dists/x/", + "https://j.example:443/dists/x/", + ] + ) + assert cacheable_https_hosts(report) == ["j.example"] + + def test_a_report_from_before_not_cached(self): + report = _report(https_sources=["https://h.example/dists/x/"]) + del report["not_cached"] + assert cacheable_https_hosts(report) == ["h.example"] + + +class TestNotCachedInTheReport: + def test_the_dump_names_them(self): + output = "\n".join( + [ + "APROXY_BEGIN", + "[dump]", + 'Acquire::http::Proxy "http://c:3142";', + 'Package-Proxy::Not-Cached "";', + 'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";', + 'Package-Proxy::Not-Cached:: "esm.ubuntu.com";', + "[sources]", + "APROXY_END", + ] + ) + assert parse_apt_proxy(output)["not_cached"] == [ + "enterprise.proxmox.com", + "esm.ubuntu.com", + ] + + def test_none(self): + assert parse_apt_proxy("APROXY_BEGIN\n[dump]\nAPROXY_END")["not_cached"] == [] + + +def test_the_hosts_are_checked(): + with pytest.raises(ValueError): + rewrite_sources_command(["h.example; rm -rf /"], "/etc/apt/apt.conf.d/00proxy") + + +_APT_CONFIG = shutil.which("apt-config") +needs_apt = pytest.mark.skipif(_APT_CONFIG is None, reason="apt-config is not on this host") + + +@pytest.fixture +def etc(tmp_path): + """apt's sources, its credentials and the proxy file, under *tmp_path*.""" + (tmp_path / "sources.list.d").mkdir() + (tmp_path / "auth.conf.d").mkdir() + (tmp_path / "00proxy").write_text('Acquire::http::Proxy "http://c:3142";\n') + return tmp_path + + +def _apt_config(etc): + return ( + f"apt-config -o Dir::Etc::sourcelist={etc}/sources.list " + f"-o Dir::Etc::sourceparts={etc}/sources.list.d/ " + f"-o Dir::Etc::netrc={etc}/auth.conf -o Dir::Etc::netrcparts={etc}/auth.conf.d/" + ) + + +def _run(command): + done = subprocess.run(["sh", "-c", command], capture_output=True, text=True, check=False) + assert done.returncode == 0, done.stderr + return done.stdout + + +_DOCKER_LIST = ( + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] " + "https://download.docker.com/linux/ubuntu noble stable\n" +) +_DOCKER_SOURCES = ( + "Types: deb\n" + "URIs: https://download.docker.com/linux/ubuntu\n" + "Suites: noble\n" + "Components: stable\n" + "Signed-By: /etc/apt/keyrings/docker.asc\n" +) + + +@needs_apt +class TestRewrite: + def test_a_one_line_source(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.list").read_text() == ( + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] " + "http://HTTPS///download.docker.com/linux/ubuntu noble stable\n" + ) + + def test_a_deb822_source(self, etc): + (etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES) + + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.sources").read_text() == ( + _DOCKER_SOURCES.replace("https://", "http://HTTPS///") + ) + + def test_sources_list_itself_and_only_the_hosts_named(self, etc): + (etc / "sources.list").write_text( + "deb http://deb.debian.org/debian trixie main\n" + "deb https://a.example/debian trixie main\n" + "deb-src https://a.example/debian trixie main\n" + "deb https://b.example/debian trixie main\n" + "# deb https://a.example/old trixie main\n" + ) + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list").read_text() == ( + "deb http://deb.debian.org/debian trixie main\n" + "deb http://HTTPS///a.example/debian trixie main\n" + "deb-src http://HTTPS///a.example/debian trixie main\n" + "deb https://b.example/debian trixie main\n" + "# deb https://a.example/old trixie main\n" + ) + + def test_a_longer_name_or_another_port_stays(self, etc): + text = ( + "deb https://a.example.org/debian trixie main\n" + "deb https://a.example:8443/debian trixie main\n" + ) + (etc / "sources.list").write_text(text) + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list").read_text() == text + + def test_a_host_with_credentials_stays_and_is_named(self, etc): + (etc / "sources.list.d" / "pve.sources").write_text( + "Types: deb\nURIs: https://enterprise.proxmox.com/debian/pve\nSuites: trixie\n" + ) + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + (etc / "auth.conf.d" / "pve.conf").write_text( + "machine https://Enterprise.Proxmox.com/debian/pve\nlogin key\npassword secret\n" + ) + + _run( + rewrite_sources_command( + ["enterprise.proxmox.com", "download.docker.com"], + f"{etc}/00proxy", + _apt_config(etc), + ) + ) + + assert "https://enterprise.proxmox.com" in (etc / "sources.list.d/pve.sources").read_text() + assert "HTTPS///download.docker.com" in (etc / "sources.list.d/docker.list").read_text() + assert (etc / "00proxy").read_text() == ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";\n' + ) + assert "secret" not in (etc / "00proxy").read_text() + + def test_credentials_in_auth_conf_on_one_line(self, etc): + (etc / "sources.list").write_text("deb https://esm.ubuntu.com/apps/ubuntu noble main\n") + (etc / "auth.conf").write_text("machine esm.ubuntu.com/apps/ubuntu/ login a password b\n") + + _run(rewrite_sources_command(["esm.ubuntu.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert "https://esm.ubuntu.com" in (etc / "sources.list").read_text() + assert "esm.ubuntu.com" in (etc / "00proxy").read_text() + + def test_a_file_without_the_host_is_left_untouched(self, etc): + other = etc / "sources.list.d" / "other.list" + other.write_text("deb http://deb.debian.org/debian trixie main\n") + before = other.stat().st_ino # sed -i writes a new file; grep first leaves it be + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert other.stat().st_ino == before + + def test_twice_is_once(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + command = rewrite_sources_command( + ["download.docker.com"], f"{etc}/00proxy", _apt_config(etc) + ) + + _run(command) + _run(command) + + assert (etc / "sources.list.d" / "docker.list").read_text().count("HTTPS///") == 1 + + +@needs_apt +class TestRevert: + def test_every_source_goes_back_to_https(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + (etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES) + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + _run(revert_sources_command(_apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.list").read_text() == _DOCKER_LIST + assert (etc / "sources.list.d" / "docker.sources").read_text() == _DOCKER_SOURCES + + def test_nothing_to_revert(self, etc): + (etc / "sources.list").write_text("deb http://deb.debian.org/debian trixie main\n") + + _run(revert_sources_command(_apt_config(etc))) + + assert ( + etc / "sources.list" + ).read_text() == "deb http://deb.debian.org/debian trixie main\n" + + +def test_the_commands_are_valid_posix_sh(): + for command in ( + rewrite_sources_command(["a.example"], "/etc/apt/apt.conf.d/00proxy"), + revert_sources_command(), + ): + assert subprocess.run(["sh", "-n", "-c", command], check=False).returncode == 0 + + +def test_a_host_name(): + assert is_host_name("download.docker.com") + assert not is_host_name('h"; Acquire::x "y') + assert not is_host_name("-h.example")