diff --git a/CHANGELOG.md b/CHANGELOG.md index a3c10b3..6ed0b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ recorded here; their history is the git log. ## [Unreleased] +### Added +- `apt_sources`: apt's https sources through the package cache, in the form apt-cacher-ng + fetches over https itself and stores (`http://HTTPS////…`, with `AllowUserPorts=80 + 443` on the cache). `cacheable_https_hosts(report)` names the hosts this applies to, and + `rewrite_sources_command` / `revert_sources_command` are the root commands, for `.list` + and deb822 `.sources` files. A host apt keeps credentials for in `auth.conf` stays on + https, because apt finds them by the host's name and the cache is shared. The rewrite + names such a host in the proxy file as `Package-Proxy::Not-Cached` (NetOrk/netork#2555). +- `get_package_proxy_report()` reports those hosts as `not_cached`. + +### Changed +- `configure_package_proxy(url)` has a step `cache` before `https`. It rewrites the https + sources it can, fetches one of them through *url*, and turns them all back to https + where *url* cannot. The `fetch` step proves the proxy with a plain http source only. +- `configure_package_proxy("")` turns the sources back to https (step `sources`) before it + removes the proxy file. Without the cache, apt would look for a host called `HTTPS`. + ## [4.39.0] - 2026-11-14 ### Added diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index daa70c3..9be8052 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -35,8 +35,12 @@ page is the right one, the caller says (``expect``). out every other line that sets a different one, reads the result back from ``apt-config``, and fetches the ``InRelease`` of one http source through it. The last step is what proves it: apt reached a mirror through the proxy it is now set -to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one -https source the same way; where the proxy refuses it, https goes ``DIRECT``. +to. The source chosen is one apt does not fetch ``DIRECT``. Then the https sources +go through the proxy in the form it stores (:mod:`~napalm_device_types.apt_sources`), +and one of them is fetched; where the proxy cannot, they go back to https. Last it +fetches one https source left the same way; where the proxy refuses it, https goes +``DIRECT``. Removing the proxy turns the sources back to https first: without the +proxy apt cannot fetch the stored form. """ from __future__ import annotations @@ -49,6 +53,14 @@ from urllib.parse import urlsplit from typing_extensions import TypedDict +from napalm_device_types.apt_sources import ( + NOT_CACHED_KEY, + VIA_CACHE, + cacheable_https_hosts, + is_host_name, + revert_sources_command, + rewrite_sources_command, +) from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps from napalm_device_types.models import FixResultDict, FixStepOutcomeDict from napalm_device_types.terminal import strip_terminal_codes @@ -87,6 +99,9 @@ class AptProxyReportDict(TypedDict): #: The base URIs of the host's sources over http and over https. http_sources: list[str] https_sources: list[str] + #: The hosts the cache rewrite left on https, for their credentials + #: (:data:`~napalm_device_types.apt_sources.NOT_CACHED_KEY`). + not_cached: list[str] class ProxyFetchDict(TypedDict): @@ -123,7 +138,8 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$") #: them early. APT_PROXY_COMMAND = ( "( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; " - "echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; " + "echo '[dump]'; apt-config dump 2>/dev/null " + "| grep -i -E '^(Acquire::https?::Proxy|Package-Proxy::)'; " 'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; ' "echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do " '[ -f "$f" ] && grep -Hin proxy "$f"; done; ' @@ -133,6 +149,7 @@ APT_PROXY_COMMAND = ( "printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat" ) +_NOT_CACHED = re.compile(rf'^{re.escape(NOT_CACHED_KEY)}::\s+"([^"]+)"', re.I) _DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I) _GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$") _FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I) @@ -293,6 +310,7 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict: "https_settings": _settings(lines, _https_setting), "http_sources": _of_scheme(sources, "http"), "https_sources": _of_scheme(sources, "https"), + "not_cached": [m.group(1) for m in map(_NOT_CACHED.match, sections.get("dump", [])) if m], } @@ -498,16 +516,20 @@ class AptProxyMixin(PackageProxyMixin): def configure_package_proxy(self, url: str) -> FixResultDict: """apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`. - https goes through *url* too, unless *url* refuses it: then ``DIRECT``. + The https sources go through *url* in the form it stores, where it can; the + rest goes through *url* too, unless *url* refuses it: then ``DIRECT``. """ if not url: - return run_fix_steps([("configure", self._remove_package_proxy)]) + return run_fix_steps( + [("sources", self._revert_sources), ("configure", self._remove_package_proxy)] + ) return run_fix_steps( [ ("configure", lambda: self._write_package_proxy(url)), ("conflicts", lambda: self._comment_out_other_proxies(url)), ("verify", lambda: self._verify_package_proxy(url)), ("fetch", lambda: self._fetch_through_package_proxy(url)), + ("cache", lambda: self._cache_https_sources(url)), ("https", lambda: self._https_through_package_proxy(url)), ] ) @@ -515,6 +537,11 @@ class AptProxyMixin(PackageProxyMixin): def _not_apt(self) -> FixStepOutcomeDict: return {"ok": False, "output": "not an apt host"} + def _revert_sources(self) -> FixStepOutcomeDict: + if not self._is_apt_host(): + return self._not_apt() + return self._run_apt_root_command(revert_sources_command(), "sources back on https") + def _remove_package_proxy(self) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() @@ -522,7 +549,9 @@ class AptProxyMixin(PackageProxyMixin): f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed" ) - def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict: + def _write_package_proxy( + self, url: str, https_direct: bool = False, not_cached: list[str] | None = None + ) -> FixStepOutcomeDict: if not self._is_apt_host(): return self._not_apt() if not PROXY_URL.match(url): @@ -530,6 +559,10 @@ class AptProxyMixin(PackageProxyMixin): content = f'Acquire::http::Proxy "{url}";\n' if https_direct: content += 'Acquire::https::Proxy "DIRECT";\n' + # Host names the report read from this file; one a line could not carry is left out. + content += "".join( + f'{NOT_CACHED_KEY}:: "{host}";\n' for host in not_cached or [] if is_host_name(host) + ) return self._write_apt_file(self.APT_PROXY_FILE, content) def _report(self) -> AptProxyReportDict: @@ -566,16 +599,7 @@ class AptProxyMixin(PackageProxyMixin): } def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict: - report = self._report() - direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} - source = next( - ( - s - for s in report["http_sources"] - if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct - ), - None, - ) + source = _plain_http_source(self._report()) if source is None: return {"ok": True, "output": "no http source to fetch through the proxy; not tried"} target = source.rstrip("/") + "/InRelease" @@ -584,6 +608,49 @@ class AptProxyMixin(PackageProxyMixin): return {"ok": True, "output": f"fetched {target} through {url}"} return {"ok": False, "output": f"{target}: {fetched['error']}"} + def _cache_https_sources(self, url: str) -> FixStepOutcomeDict: + """The https sources through *url* in the form it stores; back to https where it cannot.""" + hosts = cacheable_https_hosts(self._report()) + if not hosts: + return {"ok": True, "output": "no https source to cache"} + rewritten = self._run_apt_root_command( + rewrite_sources_command(hosts, self.APT_PROXY_FILE), "sources rewritten" + ) + if not rewritten["ok"]: + return rewritten + return self._fetch_cached(url, hosts, self._report()) + + def _fetch_cached( + self, url: str, hosts: list[str], report: AptProxyReportDict + ) -> FixStepOutcomeDict: + """One rewritten source through *url*: it stays where *url* fetched it.""" + left = sorted(set(hosts) & set(report["not_cached"])) + named = f"; left on https, apt has credentials for them: {', '.join(left)}" if left else "" + target = next( + (s.rstrip("/") + "/InRelease" for s in report["http_sources"] if _cached(s)), None + ) + if target is None: + return {"ok": True, "output": f"no https source to cache{named}"} + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target))) + if fetched["reachable"] and not fetched["error"]: + cached = ", ".join(h for h in hosts if h not in left) + return {"ok": True, "output": f"https sources cached through {url}: {cached}{named}"} + return self._back_to_https(url, f"{target}: {fetched['error']}", named) + + def _back_to_https(self, url: str, why: str, named: str) -> FixStepOutcomeDict: + """*url* did not fetch the stored form: every source goes back to https.""" + back = self._run_apt_root_command(revert_sources_command(), "sources back on https") + if not back["ok"]: + return { + "ok": False, + "output": f"{why}; turning the sources back to https failed: {back['output']}", + } + return { + "ok": True, + "output": f"{url} does not fetch https sources itself ({why}); " + f"they stay on https, not cached{named}", + } + def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict: """An https source through *url*; where *url* refuses it, https goes ``DIRECT``.""" found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url)) @@ -609,7 +676,9 @@ class AptProxyMixin(PackageProxyMixin): def _https_direct(self, url: str) -> FixStepOutcomeDict: """Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*.""" - written = self._write_package_proxy(url, https_direct=True) + written = self._write_package_proxy( + url, https_direct=True, not_cached=self._report()["not_cached"] + ) if not written["ok"]: return written later = self._later_https_lines(self._report(), url) @@ -635,6 +704,26 @@ class AptProxyMixin(PackageProxyMixin): ] +def _plain_http_source(report: AptProxyReportDict) -> str | None: + """A suite's http source the proxy takes, not in the cache's https form: its proof.""" + direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"} + return next( + ( + s + for s in report["http_sources"] + if "/dists/" in s + and not s.startswith(VIA_CACHE) + and (urlsplit(s).hostname or "").lower() not in direct + ), + None, + ) + + +def _cached(source: str) -> bool: + """A source of a suite in the form the cache stores.""" + return source.startswith(VIA_CACHE) and "/dists/" in source + + def _https_source( report: AptProxyReportDict, through: Callable[[str | None], bool] ) -> tuple[str, str] | None: diff --git a/napalm_device_types/apt_sources.py b/napalm_device_types/apt_sources.py new file mode 100644 index 0000000..2f052ce --- /dev/null +++ b/napalm_device_types/apt_sources.py @@ -0,0 +1,172 @@ +"""apt's https sources through the package cache, so that the cache stores them. + +A source fetched over https goes through apt-cacher-ng's tunnel, if at all: the +TLS runs end to end and nothing of it is stored. The cache fetches +``http://HTTPS////`` from ``https:///`` itself, with the +certificate verified, and stores it -- where it allows port 443 +(``AllowUserPorts=80 443``) (NetOrk/netork#2555). apt fetches that form through +its http proxy, so it works only while the cache is that proxy: without one apt +looks for a host called ``HTTPS``. + +**Which hosts** (:func:`cacheable_https_hosts`): the hosts of the https sources, +on port 443, while apt has an http proxy for every host and none of its own for +that one. Not a host with a login in the URL, nor one whose credentials apt keeps +in ``auth.conf``: apt finds those by the host's name, which the rewrite changes, +and paid content does not belong in a cache other hosts share. Only root reads +``auth.conf``, so the rewrite judges that, leaves such a host on https and names +it in the proxy file as ``Package-Proxy::Not-Cached``, which apt ignores and +``apt-config dump`` shows to the report. + +**The rewrite** (:func:`rewrite_sources_command`) turns ``https://`` into +the cache form on the lines that name sources: ``deb`` and ``deb-src`` in a +``.list`` file, ``URIs:`` in a deb822 ``.sources`` file. A file without the host +is not touched. :func:`revert_sources_command` turns every source back. +""" + +from __future__ import annotations + +import re +import shlex +from typing import TYPE_CHECKING +from urllib.parse import urlsplit + +if TYPE_CHECKING: # pragma: no cover - the report's type, without an import cycle + from napalm_device_types.apt_proxy import AptProxyReportDict + +#: The prefix apt-cacher-ng fetches over https. +VIA_CACHE = "http://HTTPS///" + +#: The key in the proxy file that names the hosts left on https for their credentials. +NOT_CACHED_KEY = "Package-Proxy::Not-Cached" + +_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?" +_HOST = re.compile(rf"^{_LABEL}(?:\.{_LABEL})*$") +_NO_PROXY = frozenset({"", "direct"}) + +#: Where apt's sources and credentials are, as shell variables; apt's defaults +#: when apt-config cannot say. +_LOCATE = ( + 'eval "$({apt_config} shell sl Dir::Etc::sourcelist/f sp Dir::Etc::sourceparts/d ' + 'nr Dir::Etc::netrc/f np Dir::Etc::netrcparts/d 2>/dev/null)"; ' + 'sl="${{sl:-/etc/apt/sources.list}}"; sp="${{sp:-/etc/apt/sources.list.d/}}"; ' + 'nr="${{nr:-/etc/apt/auth.conf}}"; np="${{np:-/etc/apt/auth.conf.d/}}"; ' +) + +#: The lines that name sources: ``deb``/``deb-src`` and deb822's ``URIs:``. +_SOURCE_LINES = "/^[[:space:]]*(deb|deb-src|[Uu][Rr][Ii][Ss]:)/" + +#: The hosts apt has credentials for: each word after ``machine``, without its +#: scheme, port and path, in lower case. +_CREDENTIAL_HOSTS = ( + 'creds=$(cat "$nr" "$np"* 2>/dev/null | tr -s " \\t\\r" "\\n" ' + "| awk 'p {{ print; p = 0; next }} $0 == \"machine\" {{ p = 1 }}' " + "| sed -e 's|^[A-Za-z+]*://||' -e 's|[:/].*||' | tr A-Z a-z); " +) + +_REWRITE = ( + "( " + _LOCATE + _CREDENTIAL_HOSTS + "for h in {hosts}; do " + 'l=$(printf "%s" "$h" | tr A-Z a-z); ' + 'if printf "%s\\n" "$creds" | grep -qxF -- "$l"; then ' + 'grep -qF -- "\\"$h\\"" {proxy_file} 2>/dev/null || ' + 'printf \'{key}:: "%s";\\n\' "$h" >> {proxy_file} || exit 1; continue; fi; ' + "e=$(printf '%s' \"$h\" | sed 's/[.]/[.]/g'); " + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do ' + '[ -f "$f" ] && grep -qF -- "https://$h" "$f" || continue; ' + 'sed -i -E "{lines} s#([[:space:]:])https://$e([/[:space:]]|\\$)#\\1{via}$h\\2#g" "$f" ' + "|| exit 1; done; done )" +) + +_REVERT = ( + "( " + _LOCATE + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do ' + '[ -f "$f" ] && grep -qF -- "{via}" "$f" || continue; ' + "sed -i -E '{lines} s#([[:space:]:]){via}#\\1https://#g' \"$f\" || exit 1; done )" +) + + +def cache_form(url: str) -> str: + """*url*, an https URL, in the form apt-cacher-ng fetches over https and stores. + + :raises ValueError: if *url* is not an https URL. + """ + if not url.startswith("https://"): + raise ValueError(f"not an https URL: {url!r}") + return VIA_CACHE + url[len("https://") :] + + +def _per_host_direct(report: AptProxyReportDict) -> set[str]: + hosts = {**report["per_host"], **report["https_per_host"]} + return {h.lower() for h, v in hosts.items() if v.lower() in _NO_PROXY} + + +def _host_of(source: str) -> str | None: + """The host of an https source the cache can fetch: port 443, no login in the URL.""" + try: + parts = urlsplit(source) + port = parts.port + except ValueError: + return None + if parts.username is not None or port not in (None, 443) or not parts.hostname: + return None + return parts.hostname + + +def cacheable_https_hosts(report: AptProxyReportDict) -> list[str]: + """The hosts of the https sources the rewrite would send through the cache, sorted. + + None while apt has no http proxy for every host: the cache form goes through it. + Not a host apt fetches ``DIRECT``, nor one the rewrite left on https for its + credentials (``not_cached``), nor one on another port or with a login in the URL. + """ + if not report["proxy"]: + return [] + skip = _per_host_direct(report) | {h.lower() for h in report.get("not_cached", [])} + hosts = {_host_of(s) for s in report["https_sources"]} + return sorted(h for h in hosts if h and h.lower() not in skip) + + +def is_host_name(host: str) -> bool: + """Whether *host* is a DNS name: letters, digits and hyphens between dots.""" + return bool(_HOST.match(host)) + + +def _checked(hosts: list[str]) -> str: + for host in hosts: + if not is_host_name(host): + raise ValueError(f"not a host name: {host!r}") + return " ".join(hosts) + + +def rewrite_sources_command( + hosts: list[str], proxy_file: str, apt_config: str = "apt-config" +) -> str: + """The root command that sends the https sources of *hosts* through the cache. + + A host apt has credentials for stays on https and is named in *proxy_file*. + *apt_config* is how apt-config is called; a test points apt's paths elsewhere. + + :raises ValueError: if one of *hosts* is not a host name. + """ + return _REWRITE.format( + apt_config=apt_config, + hosts=_checked(hosts), + proxy_file=shlex.quote(proxy_file), + key=NOT_CACHED_KEY, + lines=_SOURCE_LINES, + via=VIA_CACHE, + ) + + +def revert_sources_command(apt_config: str = "apt-config") -> str: + """The root command that turns every source in the cache form back to https.""" + return _REVERT.format(apt_config=apt_config, lines=_SOURCE_LINES, via=VIA_CACHE) + + +__all__ = [ + "NOT_CACHED_KEY", + "VIA_CACHE", + "cache_form", + "cacheable_https_hosts", + "is_host_name", + "revert_sources_command", + "rewrite_sources_command", +] diff --git a/tests/test_apt_proxy.py b/tests/test_apt_proxy.py index b886211..d8f8da8 100644 --- a/tests/test_apt_proxy.py +++ b/tests/test_apt_proxy.py @@ -25,6 +25,7 @@ from napalm_device_types.apt_proxy import ( parse_apt_proxy, same_proxy_url, ) +from napalm_device_types.apt_sources import NOT_CACHED_KEY, VIA_CACHE from napalm_device_types.fixes import PackageProxyMixin @@ -302,6 +303,13 @@ _CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/ E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142] E: Download Failed""" +#: A cache that does not fetch the cache form itself: ``AllowUserPorts`` is 80 alone +#: (apt-cacher-ng 3.7.5, NetOrk/netork#2555). +_PROHIBITED_PORT = """Err:1 http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease + 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143] +E: Failed to fetch http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143] +E: Download Failed""" + def _fetch(output, rc, match=None): lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)] @@ -422,9 +430,21 @@ class _Host(AptProxyMixin): """ def __init__( - self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0 + self, + files=None, + fetch_rc=0, + apt=True, + https_sources=(), + tunnel=True, + direct_rc=0, + cache_https=False, + credentials=(), ): self.files = dict(files or {}) + #: Whether the proxy fetches the cache form itself (AllowUserPorts 443), and + #: the hosts apt keeps credentials for. + self.cache_https = cache_https + self.credentials = set(credentials) self.fetch_rc = fetch_rc self.apt = apt self.https_sources = list(https_sources) @@ -459,6 +479,12 @@ class _Host(AptProxyMixin): value, https = self._effective(), self._effective("https") dump = [f'Acquire::http::Proxy "{value}";'] if value else [] dump += [f'Acquire::https::Proxy "{https}";'] if https else [] + dump += [ + line + for path, text in self.files.items() + for line in text.splitlines() + if line.startswith(NOT_CACHED_KEY) + ] return _report( dump=dump, parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)], @@ -466,6 +492,8 @@ class _Host(AptProxyMixin): sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources], ) self.fetched.append(command) + if re.search(r"download-file '?http://HTTPS///", command): + return _fetch(_FETCHED, 0) if self.cache_https else _fetch(_PROHIBITED_PORT, 100) if re.search(r"download-file '?https://", command): return self._fetch_https(command) return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1) @@ -482,6 +510,11 @@ class _Host(AptProxyMixin): def _run_apt_root_command(self, command, done): self.root.append(command) + if NOT_CACHED_KEY in command: + return self._rewrite(re.search(r"for h in ([^;]+); do", command).group(1).split(), done) + if "#\\1https://#g" in command: + self.https_sources = [s.replace(VIA_CACHE, "https://") for s in self.https_sources] + return {"ok": True, "output": done} path = command.rsplit(" ", 1)[-1].strip("'") for m in re.finditer(r"(\d+)\{/[^}]*\}", command): n = int(m.group(1)) @@ -492,6 +525,19 @@ class _Host(AptProxyMixin): self.files.pop(command.split()[-1].strip("'"), None) return {"ok": True, "output": done} + def _rewrite(self, hosts, done): + for host in hosts: + if host in self.credentials: + proxy = self.files.get(f"{_D}00proxy", "") + self.files[f"{_D}00proxy"] = proxy + f'{NOT_CACHED_KEY}:: "{host}";\n' + continue + prefix = f"https://{host}/" + self.https_sources = [ + VIA_CACHE + s[len("https://") :] if s.startswith(prefix) else s + for s in self.https_sources + ] + return {"ok": True, "output": done} + class TestCheckHttpsSource: def test_one_source_the_way_apt_fetches_it(self): @@ -569,8 +615,10 @@ class TestConfigurePackageProxy: "conflicts", "verify", "fetch", + "cache", "https", ] + assert "no https source" in _step(result, "cache")["output"] assert "no https source" in result["steps"][-1]["output"] assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n' assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0] @@ -766,6 +814,144 @@ class TestConfigurePackageProxy: assert host.files == {} +class TestCacheHttpsSources: + """NetOrk/netork#2555: the cache stores an https source only in its own form.""" + + def test_a_cache_that_fetches_https_gets_them(self): + host = _Host(https_sources=[_DOCKER], cache_https=True) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.https_sources == [f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"] + assert _step(result, "cache")["output"] == ( + "https sources cached through http://c:3142: download.docker.com" + ) + assert any(VIA_CACHE in c for c in host.fetched) + assert "no https source goes through" in _step(result, "https")["output"] + + def test_a_cache_that_does_not_turns_them_back(self): + host = _Host(https_sources=[_DOCKER]) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.https_sources == [_DOCKER] + output = _step(result, "cache")["output"] + assert "http://c:3142 does not fetch https sources itself" in output + assert "prohibited port (see AllowUserPorts)" in output + assert "they stay on https, not cached" in output + assert _step(result, "https")["output"] == ( + f"fetched {_DOCKER}InRelease through http://c:3142" + ) + + def test_a_host_with_credentials_stays_on_https_and_is_named(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host( + https_sources=[pve, _DOCKER], + cache_https=True, + credentials={"enterprise.proxmox.com"}, + ) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert pve in host.https_sources + output = _step(result, "cache")["output"] + assert "cached through http://c:3142: download.docker.com" in output + assert "left on https, apt has credentials for them: enterprise.proxmox.com" in output + assert host.get_package_proxy_report()["not_cached"] == ["enterprise.proxmox.com"] + + def test_only_hosts_with_credentials(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host(https_sources=[pve], cache_https=True, credentials={"enterprise.proxmox.com"}) + + result = host.configure_package_proxy("http://c:3142") + + assert _step(result, "cache")["output"] == ( + "no https source to cache; " + "left on https, apt has credentials for them: enterprise.proxmox.com" + ) + + def test_the_https_direct_fallback_keeps_the_hosts_left_on_https(self): + pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/" + host = _Host(https_sources=[pve], tunnel=False, credentials={"enterprise.proxmox.com"}) + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is True + assert host.files[f"{_D}00proxy"] == ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Acquire::https::Proxy "DIRECT";\n' + f'{NOT_CACHED_KEY}:: "enterprise.proxmox.com";\n' + ) + + def test_a_rewrite_that_fails_stops_the_fix(self): + host = _Host(https_sources=[_DOCKER], cache_https=True) + real = host._run_apt_root_command + + def denied(command, done): + if NOT_CACHED_KEY in command: + return {"ok": False, "output": "sed: denied"} + return real(command, done) + + host._run_apt_root_command = denied + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + assert result["steps"][-1] == {"step": "cache", "ok": False, "output": "sed: denied"} + + def test_a_turn_back_that_fails_says_so(self): + host = _Host(https_sources=[_DOCKER]) + real = host._run_apt_root_command + + def denied(command, done): + if "#\\1https://#g" in command: + return {"ok": False, "output": "sed: denied"} + return real(command, done) + + host._run_apt_root_command = denied + + result = host.configure_package_proxy("http://c:3142") + + assert result["success"] is False + output = result["steps"][-1]["output"] + assert "turning the sources back to https failed: sed: denied" in output + + def test_the_proof_of_the_proxy_is_a_plain_http_source(self): + """A source already in the cache form says nothing about plain http.""" + host = _Host(https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"]) + + host.configure_package_proxy("http://c:3142") + + fetch = next(c for c in host.fetched if "InRelease" in c) + assert "deb.debian.org" in fetch + + def test_removing_the_proxy_turns_the_sources_back_first(self): + host = _Host( + {f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}, + https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"], + ) + + result = host.configure_package_proxy("") + + assert result["success"] is True + assert [s["step"] for s in result["steps"]] == ["sources", "configure"] + assert host.https_sources == [_DOCKER] + assert host.files == {} + + def test_a_turn_back_that_fails_keeps_the_proxy(self): + """Without the cache apt looks for a host called HTTPS: the proxy stays.""" + host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'}) + host._run_apt_root_command = lambda command, done: {"ok": False, "output": "denied"} + + result = host.configure_package_proxy("") + + assert result["success"] is False + assert f"{_D}00proxy" in host.files + + class TestWhatTheParserSkips: def test_a_line_that_is_no_grep_output(self): assert ( @@ -824,3 +1010,11 @@ def test_the_comment_out_command_for_https(): assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == ( "sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf" ) + + +def test_a_line_left_on_https_that_is_no_host_name_is_not_written_back(): + host = _Host() + host._write_package_proxy("http://c:3142", not_cached=["ok.example", 'x"; Foo "1']) + assert host.files[f"{_D}00proxy"] == ( + f'Acquire::http::Proxy "http://c:3142";\n{NOT_CACHED_KEY}:: "ok.example";\n' + ) diff --git a/tests/test_apt_sources.py b/tests/test_apt_sources.py new file mode 100644 index 0000000..5db305c --- /dev/null +++ b/tests/test_apt_sources.py @@ -0,0 +1,318 @@ +"""https sources through the cache: which can be, and the root commands that turn them. + +apt-cacher-ng fetches ``http://HTTPS////`` from ``https:///`` +and stores it, where it allows port 443 (``AllowUserPorts=80 443``); a source +fetched over https itself goes through the cache's tunnel and is never stored +(NetOrk/netork#2555). The commands run here for real, on files in a temporary +directory that ``apt-config -o`` points apt's own settings at. +""" + +from __future__ import annotations + +import shutil +import subprocess + +import pytest + +from napalm_device_types.apt_proxy import parse_apt_proxy +from napalm_device_types.apt_sources import ( + VIA_CACHE, + cache_form, + cacheable_https_hosts, + is_host_name, + revert_sources_command, + rewrite_sources_command, +) + + +def _report(**over): + report = { + "proxy": "http://c:3142", + "per_host": {}, + "auto_detect": None, + "settings": [], + "https_proxy": "http://c:3142", + "https_per_host": {}, + "https_settings": [], + "http_sources": ["http://deb.debian.org/debian/dists/trixie/"], + "https_sources": [], + "not_cached": [], + } + report.update(over) + return report + + +class TestCacheForm: + def test_an_https_url(self): + assert cache_form("https://download.docker.com/linux/ubuntu/dists/noble/") == ( + "http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/" + ) + + def test_the_prefix(self): + assert VIA_CACHE == "http://HTTPS///" + + def test_not_an_https_url(self): + with pytest.raises(ValueError): + cache_form("http://deb.debian.org/debian/") + + +class TestCacheableHttpsHosts: + def test_each_host_once_sorted(self): + report = _report( + https_sources=[ + "https://packages.wazuh.com/4.x/apt/dists/stable/", + "https://download.docker.com/linux/ubuntu/dists/noble/", + "https://download.docker.com/linux/ubuntu/dists/noble-test/", + ] + ) + assert cacheable_https_hosts(report) == ["download.docker.com", "packages.wazuh.com"] + + def test_none_without_an_http_proxy(self): + # The cache form goes through the http proxy: without one it goes nowhere. + report = _report(proxy=None, https_sources=["https://h.example/dists/x/"]) + assert cacheable_https_hosts(report) == [] + + def test_a_host_apt_fetches_direct_stays(self): + report = _report( + per_host={"a.example": "DIRECT"}, + https_per_host={"B.example": "direct"}, + https_sources=[ + "https://a.example/dists/x/", + "https://b.example/dists/x/", + "https://c.example/dists/x/", + ], + ) + assert cacheable_https_hosts(report) == ["c.example"] + + def test_a_host_with_credentials_stays(self): + report = _report( + https_sources=["https://enterprise.proxmox.com/debian/pve/dists/trixie/"], + not_cached=["enterprise.proxmox.com"], + ) + assert cacheable_https_hosts(report) == [] + + def test_another_port_or_a_login_in_the_url_stays(self): + # The cache allows port 443 alone, and a login in the URL is a credential too. + report = _report( + https_sources=[ + "https://h.example:8443/dists/x/", + "https://u:p@i.example/dists/x/", + "https://j.example:443/dists/x/", + ] + ) + assert cacheable_https_hosts(report) == ["j.example"] + + def test_a_report_from_before_not_cached(self): + report = _report(https_sources=["https://h.example/dists/x/"]) + del report["not_cached"] + assert cacheable_https_hosts(report) == ["h.example"] + + +class TestNotCachedInTheReport: + def test_the_dump_names_them(self): + output = "\n".join( + [ + "APROXY_BEGIN", + "[dump]", + 'Acquire::http::Proxy "http://c:3142";', + 'Package-Proxy::Not-Cached "";', + 'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";', + 'Package-Proxy::Not-Cached:: "esm.ubuntu.com";', + "[sources]", + "APROXY_END", + ] + ) + assert parse_apt_proxy(output)["not_cached"] == [ + "enterprise.proxmox.com", + "esm.ubuntu.com", + ] + + def test_none(self): + assert parse_apt_proxy("APROXY_BEGIN\n[dump]\nAPROXY_END")["not_cached"] == [] + + +def test_the_hosts_are_checked(): + with pytest.raises(ValueError): + rewrite_sources_command(["h.example; rm -rf /"], "/etc/apt/apt.conf.d/00proxy") + + +_APT_CONFIG = shutil.which("apt-config") +needs_apt = pytest.mark.skipif(_APT_CONFIG is None, reason="apt-config is not on this host") + + +@pytest.fixture +def etc(tmp_path): + """apt's sources, its credentials and the proxy file, under *tmp_path*.""" + (tmp_path / "sources.list.d").mkdir() + (tmp_path / "auth.conf.d").mkdir() + (tmp_path / "00proxy").write_text('Acquire::http::Proxy "http://c:3142";\n') + return tmp_path + + +def _apt_config(etc): + return ( + f"apt-config -o Dir::Etc::sourcelist={etc}/sources.list " + f"-o Dir::Etc::sourceparts={etc}/sources.list.d/ " + f"-o Dir::Etc::netrc={etc}/auth.conf -o Dir::Etc::netrcparts={etc}/auth.conf.d/" + ) + + +def _run(command): + done = subprocess.run(["sh", "-c", command], capture_output=True, text=True, check=False) + assert done.returncode == 0, done.stderr + return done.stdout + + +_DOCKER_LIST = ( + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] " + "https://download.docker.com/linux/ubuntu noble stable\n" +) +_DOCKER_SOURCES = ( + "Types: deb\n" + "URIs: https://download.docker.com/linux/ubuntu\n" + "Suites: noble\n" + "Components: stable\n" + "Signed-By: /etc/apt/keyrings/docker.asc\n" +) + + +@needs_apt +class TestRewrite: + def test_a_one_line_source(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.list").read_text() == ( + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] " + "http://HTTPS///download.docker.com/linux/ubuntu noble stable\n" + ) + + def test_a_deb822_source(self, etc): + (etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES) + + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.sources").read_text() == ( + _DOCKER_SOURCES.replace("https://", "http://HTTPS///") + ) + + def test_sources_list_itself_and_only_the_hosts_named(self, etc): + (etc / "sources.list").write_text( + "deb http://deb.debian.org/debian trixie main\n" + "deb https://a.example/debian trixie main\n" + "deb-src https://a.example/debian trixie main\n" + "deb https://b.example/debian trixie main\n" + "# deb https://a.example/old trixie main\n" + ) + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list").read_text() == ( + "deb http://deb.debian.org/debian trixie main\n" + "deb http://HTTPS///a.example/debian trixie main\n" + "deb-src http://HTTPS///a.example/debian trixie main\n" + "deb https://b.example/debian trixie main\n" + "# deb https://a.example/old trixie main\n" + ) + + def test_a_longer_name_or_another_port_stays(self, etc): + text = ( + "deb https://a.example.org/debian trixie main\n" + "deb https://a.example:8443/debian trixie main\n" + ) + (etc / "sources.list").write_text(text) + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert (etc / "sources.list").read_text() == text + + def test_a_host_with_credentials_stays_and_is_named(self, etc): + (etc / "sources.list.d" / "pve.sources").write_text( + "Types: deb\nURIs: https://enterprise.proxmox.com/debian/pve\nSuites: trixie\n" + ) + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + (etc / "auth.conf.d" / "pve.conf").write_text( + "machine https://Enterprise.Proxmox.com/debian/pve\nlogin key\npassword secret\n" + ) + + _run( + rewrite_sources_command( + ["enterprise.proxmox.com", "download.docker.com"], + f"{etc}/00proxy", + _apt_config(etc), + ) + ) + + assert "https://enterprise.proxmox.com" in (etc / "sources.list.d/pve.sources").read_text() + assert "HTTPS///download.docker.com" in (etc / "sources.list.d/docker.list").read_text() + assert (etc / "00proxy").read_text() == ( + 'Acquire::http::Proxy "http://c:3142";\n' + 'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";\n' + ) + assert "secret" not in (etc / "00proxy").read_text() + + def test_credentials_in_auth_conf_on_one_line(self, etc): + (etc / "sources.list").write_text("deb https://esm.ubuntu.com/apps/ubuntu noble main\n") + (etc / "auth.conf").write_text("machine esm.ubuntu.com/apps/ubuntu/ login a password b\n") + + _run(rewrite_sources_command(["esm.ubuntu.com"], f"{etc}/00proxy", _apt_config(etc))) + + assert "https://esm.ubuntu.com" in (etc / "sources.list").read_text() + assert "esm.ubuntu.com" in (etc / "00proxy").read_text() + + def test_a_file_without_the_host_is_left_untouched(self, etc): + other = etc / "sources.list.d" / "other.list" + other.write_text("deb http://deb.debian.org/debian trixie main\n") + before = other.stat().st_ino # sed -i writes a new file; grep first leaves it be + + _run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc))) + + assert other.stat().st_ino == before + + def test_twice_is_once(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + command = rewrite_sources_command( + ["download.docker.com"], f"{etc}/00proxy", _apt_config(etc) + ) + + _run(command) + _run(command) + + assert (etc / "sources.list.d" / "docker.list").read_text().count("HTTPS///") == 1 + + +@needs_apt +class TestRevert: + def test_every_source_goes_back_to_https(self, etc): + (etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST) + (etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES) + _run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc))) + + _run(revert_sources_command(_apt_config(etc))) + + assert (etc / "sources.list.d" / "docker.list").read_text() == _DOCKER_LIST + assert (etc / "sources.list.d" / "docker.sources").read_text() == _DOCKER_SOURCES + + def test_nothing_to_revert(self, etc): + (etc / "sources.list").write_text("deb http://deb.debian.org/debian trixie main\n") + + _run(revert_sources_command(_apt_config(etc))) + + assert ( + etc / "sources.list" + ).read_text() == "deb http://deb.debian.org/debian trixie main\n" + + +def test_the_commands_are_valid_posix_sh(): + for command in ( + rewrite_sources_command(["a.example"], "/etc/apt/apt.conf.d/00proxy"), + revert_sources_command(), + ): + assert subprocess.run(["sh", "-n", "-c", command], check=False).returncode == 0 + + +def test_a_host_name(): + assert is_host_name("download.docker.com") + assert not is_host_name('h"; Acquire::x "y') + assert not is_host_name("-h.example")