feat(apt): send https sources through the cache in the form it stores (NetOrk/netork#2555)
driver / lint (pull_request) Successful in 15s
driver / standard (pull_request) Successful in 19s
driver / types (pull_request) Successful in 58s
driver / test (3.13) (pull_request) Successful in 1m1s
driver / test (3.12) (pull_request) Successful in 1m8s
driver / lowest (pull_request) Successful in 35s
driver / build (pull_request) Successful in 29s
driver / security (pull_request) Successful in 1m9s
CI / driver (pull_request) Successful in 2m29s
driver / publish (pull_request) Skipped

apt-cacher-ng tunnels a source fetched over https and stores nothing of it. It
fetches http://HTTPS///<host>/<path> over https itself, with the certificate
verified, and stores it, where port 443 is allowed (AllowUserPorts=80 443).

- apt_sources: cacheable_https_hosts(report), and the root commands that rewrite
  the https sources of .list and deb822 .sources files to that form and back.
  A host apt has credentials for in auth.conf stays on https; the rewrite names
  it in the proxy file as Package-Proxy::Not-Cached, which the report reads.
- configure_package_proxy(url): a step "cache" before "https". It rewrites the
  sources, fetches one of them through url, and turns them all back to https
  where url cannot. The "fetch" step proves the proxy with a plain http source.
- configure_package_proxy(""): the sources go back to https before the proxy
  file goes; without the cache apt would look for a host called HTTPS.
This commit is contained in:
christianmanivong committed 2026-11-15 20:07:27 +01:00
1 parent b4929990f1
commit cb0f1cc05c
5 files changed
+808 -18

No files matched your search

+106 -17
View File
@@ -35,8 +35,12 @@ page is the right one, the caller says (``expect``).
out every other line that sets a different one, reads the result back from
``apt-config``, and fetches the ``InRelease`` of one http source through it. The
last step is what proves it: apt reached a mirror through the proxy it is now set
to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one
https source the same way; where the proxy refuses it, https goes ``DIRECT``.
to. The source chosen is one apt does not fetch ``DIRECT``. Then the https sources
go through the proxy in the form it stores (:mod:`~napalm_device_types.apt_sources`),
and one of them is fetched; where the proxy cannot, they go back to https. Last it
fetches one https source left the same way; where the proxy refuses it, https goes
``DIRECT``. Removing the proxy turns the sources back to https first: without the
proxy apt cannot fetch the stored form.
"""
from __future__ import annotations
@@ -49,6 +53,14 @@ from urllib.parse import urlsplit
from typing_extensions import TypedDict
from napalm_device_types.apt_sources import (
NOT_CACHED_KEY,
VIA_CACHE,
cacheable_https_hosts,
is_host_name,
revert_sources_command,
rewrite_sources_command,
)
from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps
from napalm_device_types.models import FixResultDict, FixStepOutcomeDict
from napalm_device_types.terminal import strip_terminal_codes
@@ -87,6 +99,9 @@ class AptProxyReportDict(TypedDict):
#: The base URIs of the host's sources over http and over https.
http_sources: list[str]
https_sources: list[str]
#: The hosts the cache rewrite left on https, for their credentials
#: (:data:`~napalm_device_types.apt_sources.NOT_CACHED_KEY`).
not_cached: list[str]
class ProxyFetchDict(TypedDict):
@@ -123,7 +138,8 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$")
#: them early.
APT_PROXY_COMMAND = (
"( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; "
"echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; "
"echo '[dump]'; apt-config dump 2>/dev/null "
"| grep -i -E '^(Acquire::https?::Proxy|Package-Proxy::)'; "
'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; '
"echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do "
'[ -f "$f" ] && grep -Hin proxy "$f"; done; '
@@ -133,6 +149,7 @@ APT_PROXY_COMMAND = (
"printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat"
)
_NOT_CACHED = re.compile(rf'^{re.escape(NOT_CACHED_KEY)}::\s+"([^"]+)"', re.I)
_DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I)
_GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$")
_FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I)
@@ -293,6 +310,7 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict:
"https_settings": _settings(lines, _https_setting),
"http_sources": _of_scheme(sources, "http"),
"https_sources": _of_scheme(sources, "https"),
"not_cached": [m.group(1) for m in map(_NOT_CACHED.match, sections.get("dump", [])) if m],
}
@@ -498,16 +516,20 @@ class AptProxyMixin(PackageProxyMixin):
def configure_package_proxy(self, url: str) -> FixResultDict:
"""apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`.
https goes through *url* too, unless *url* refuses it: then ``DIRECT``.
The https sources go through *url* in the form it stores, where it can; the
rest goes through *url* too, unless *url* refuses it: then ``DIRECT``.
"""
if not url:
return run_fix_steps([("configure", self._remove_package_proxy)])
return run_fix_steps(
[("sources", self._revert_sources), ("configure", self._remove_package_proxy)]
)
return run_fix_steps(
[
("configure", lambda: self._write_package_proxy(url)),
("conflicts", lambda: self._comment_out_other_proxies(url)),
("verify", lambda: self._verify_package_proxy(url)),
("fetch", lambda: self._fetch_through_package_proxy(url)),
("cache", lambda: self._cache_https_sources(url)),
("https", lambda: self._https_through_package_proxy(url)),
]
)
@@ -515,6 +537,11 @@ class AptProxyMixin(PackageProxyMixin):
def _not_apt(self) -> FixStepOutcomeDict:
return {"ok": False, "output": "not an apt host"}
def _revert_sources(self) -> FixStepOutcomeDict:
if not self._is_apt_host():
return self._not_apt()
return self._run_apt_root_command(revert_sources_command(), "sources back on https")
def _remove_package_proxy(self) -> FixStepOutcomeDict:
if not self._is_apt_host():
return self._not_apt()
@@ -522,7 +549,9 @@ class AptProxyMixin(PackageProxyMixin):
f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed"
)
def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict:
def _write_package_proxy(
self, url: str, https_direct: bool = False, not_cached: list[str] | None = None
) -> FixStepOutcomeDict:
if not self._is_apt_host():
return self._not_apt()
if not PROXY_URL.match(url):
@@ -530,6 +559,10 @@ class AptProxyMixin(PackageProxyMixin):
content = f'Acquire::http::Proxy "{url}";\n'
if https_direct:
content += 'Acquire::https::Proxy "DIRECT";\n'
# Host names the report read from this file; one a line could not carry is left out.
content += "".join(
f'{NOT_CACHED_KEY}:: "{host}";\n' for host in not_cached or [] if is_host_name(host)
)
return self._write_apt_file(self.APT_PROXY_FILE, content)
def _report(self) -> AptProxyReportDict:
@@ -566,16 +599,7 @@ class AptProxyMixin(PackageProxyMixin):
}
def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict:
report = self._report()
direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"}
source = next(
(
s
for s in report["http_sources"]
if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct
),
None,
)
source = _plain_http_source(self._report())
if source is None:
return {"ok": True, "output": "no http source to fetch through the proxy; not tried"}
target = source.rstrip("/") + "/InRelease"
@@ -584,6 +608,49 @@ class AptProxyMixin(PackageProxyMixin):
return {"ok": True, "output": f"fetched {target} through {url}"}
return {"ok": False, "output": f"{target}: {fetched['error']}"}
def _cache_https_sources(self, url: str) -> FixStepOutcomeDict:
"""The https sources through *url* in the form it stores; back to https where it cannot."""
hosts = cacheable_https_hosts(self._report())
if not hosts:
return {"ok": True, "output": "no https source to cache"}
rewritten = self._run_apt_root_command(
rewrite_sources_command(hosts, self.APT_PROXY_FILE), "sources rewritten"
)
if not rewritten["ok"]:
return rewritten
return self._fetch_cached(url, hosts, self._report())
def _fetch_cached(
self, url: str, hosts: list[str], report: AptProxyReportDict
) -> FixStepOutcomeDict:
"""One rewritten source through *url*: it stays where *url* fetched it."""
left = sorted(set(hosts) & set(report["not_cached"]))
named = f"; left on https, apt has credentials for them: {', '.join(left)}" if left else ""
target = next(
(s.rstrip("/") + "/InRelease" for s in report["http_sources"] if _cached(s)), None
)
if target is None:
return {"ok": True, "output": f"no https source to cache{named}"}
fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target)))
if fetched["reachable"] and not fetched["error"]:
cached = ", ".join(h for h in hosts if h not in left)
return {"ok": True, "output": f"https sources cached through {url}: {cached}{named}"}
return self._back_to_https(url, f"{target}: {fetched['error']}", named)
def _back_to_https(self, url: str, why: str, named: str) -> FixStepOutcomeDict:
"""*url* did not fetch the stored form: every source goes back to https."""
back = self._run_apt_root_command(revert_sources_command(), "sources back on https")
if not back["ok"]:
return {
"ok": False,
"output": f"{why}; turning the sources back to https failed: {back['output']}",
}
return {
"ok": True,
"output": f"{url} does not fetch https sources itself ({why}); "
f"they stay on https, not cached{named}",
}
def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict:
"""An https source through *url*; where *url* refuses it, https goes ``DIRECT``."""
found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url))
@@ -609,7 +676,9 @@ class AptProxyMixin(PackageProxyMixin):
def _https_direct(self, url: str) -> FixStepOutcomeDict:
"""Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*."""
written = self._write_package_proxy(url, https_direct=True)
written = self._write_package_proxy(
url, https_direct=True, not_cached=self._report()["not_cached"]
)
if not written["ok"]:
return written
later = self._later_https_lines(self._report(), url)
@@ -635,6 +704,26 @@ class AptProxyMixin(PackageProxyMixin):
]
def _plain_http_source(report: AptProxyReportDict) -> str | None:
"""A suite's http source the proxy takes, not in the cache's https form: its proof."""
direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"}
return next(
(
s
for s in report["http_sources"]
if "/dists/" in s
and not s.startswith(VIA_CACHE)
and (urlsplit(s).hostname or "").lower() not in direct
),
None,
)
def _cached(source: str) -> bool:
"""A source of a suite in the form the cache stores."""
return source.startswith(VIA_CACHE) and "/dists/" in source
def _https_source(
report: AptProxyReportDict, through: Callable[[str | None], bool]
) -> tuple[str, str] | None:
+172
View File
@@ -0,0 +1,172 @@
"""apt's https sources through the package cache, so that the cache stores them.
A source fetched over https goes through apt-cacher-ng's tunnel, if at all: the
TLS runs end to end and nothing of it is stored. The cache fetches
``http://HTTPS///<host>/<path>`` from ``https://<host>/<path>`` itself, with the
certificate verified, and stores it -- where it allows port 443
(``AllowUserPorts=80 443``) (NetOrk/netork#2555). apt fetches that form through
its http proxy, so it works only while the cache is that proxy: without one apt
looks for a host called ``HTTPS``.
**Which hosts** (:func:`cacheable_https_hosts`): the hosts of the https sources,
on port 443, while apt has an http proxy for every host and none of its own for
that one. Not a host with a login in the URL, nor one whose credentials apt keeps
in ``auth.conf``: apt finds those by the host's name, which the rewrite changes,
and paid content does not belong in a cache other hosts share. Only root reads
``auth.conf``, so the rewrite judges that, leaves such a host on https and names
it in the proxy file as ``Package-Proxy::Not-Cached``, which apt ignores and
``apt-config dump`` shows to the report.
**The rewrite** (:func:`rewrite_sources_command`) turns ``https://<host>`` into
the cache form on the lines that name sources: ``deb`` and ``deb-src`` in a
``.list`` file, ``URIs:`` in a deb822 ``.sources`` file. A file without the host
is not touched. :func:`revert_sources_command` turns every source back.
"""
from __future__ import annotations
import re
import shlex
from typing import TYPE_CHECKING
from urllib.parse import urlsplit
if TYPE_CHECKING: # pragma: no cover - the report's type, without an import cycle
from napalm_device_types.apt_proxy import AptProxyReportDict
#: The prefix apt-cacher-ng fetches over https.
VIA_CACHE = "http://HTTPS///"
#: The key in the proxy file that names the hosts left on https for their credentials.
NOT_CACHED_KEY = "Package-Proxy::Not-Cached"
_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?"
_HOST = re.compile(rf"^{_LABEL}(?:\.{_LABEL})*$")
_NO_PROXY = frozenset({"", "direct"})
#: Where apt's sources and credentials are, as shell variables; apt's defaults
#: when apt-config cannot say.
_LOCATE = (
'eval "$({apt_config} shell sl Dir::Etc::sourcelist/f sp Dir::Etc::sourceparts/d '
'nr Dir::Etc::netrc/f np Dir::Etc::netrcparts/d 2>/dev/null)"; '
'sl="${{sl:-/etc/apt/sources.list}}"; sp="${{sp:-/etc/apt/sources.list.d/}}"; '
'nr="${{nr:-/etc/apt/auth.conf}}"; np="${{np:-/etc/apt/auth.conf.d/}}"; '
)
#: The lines that name sources: ``deb``/``deb-src`` and deb822's ``URIs:``.
_SOURCE_LINES = "/^[[:space:]]*(deb|deb-src|[Uu][Rr][Ii][Ss]:)/"
#: The hosts apt has credentials for: each word after ``machine``, without its
#: scheme, port and path, in lower case.
_CREDENTIAL_HOSTS = (
'creds=$(cat "$nr" "$np"* 2>/dev/null | tr -s " \\t\\r" "\\n" '
"| awk 'p {{ print; p = 0; next }} $0 == \"machine\" {{ p = 1 }}' "
"| sed -e 's|^[A-Za-z+]*://||' -e 's|[:/].*||' | tr A-Z a-z); "
)
_REWRITE = (
"( " + _LOCATE + _CREDENTIAL_HOSTS + "for h in {hosts}; do "
'l=$(printf "%s" "$h" | tr A-Z a-z); '
'if printf "%s\\n" "$creds" | grep -qxF -- "$l"; then '
'grep -qF -- "\\"$h\\"" {proxy_file} 2>/dev/null || '
'printf \'{key}:: "%s";\\n\' "$h" >> {proxy_file} || exit 1; continue; fi; '
"e=$(printf '%s' \"$h\" | sed 's/[.]/[.]/g'); "
'for f in "$sl" "$sp"*.list "$sp"*.sources; do '
'[ -f "$f" ] && grep -qF -- "https://$h" "$f" || continue; '
'sed -i -E "{lines} s#([[:space:]:])https://$e([/[:space:]]|\\$)#\\1{via}$h\\2#g" "$f" '
"|| exit 1; done; done )"
)
_REVERT = (
"( " + _LOCATE + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do '
'[ -f "$f" ] && grep -qF -- "{via}" "$f" || continue; '
"sed -i -E '{lines} s#([[:space:]:]){via}#\\1https://#g' \"$f\" || exit 1; done )"
)
def cache_form(url: str) -> str:
"""*url*, an https URL, in the form apt-cacher-ng fetches over https and stores.
:raises ValueError: if *url* is not an https URL.
"""
if not url.startswith("https://"):
raise ValueError(f"not an https URL: {url!r}")
return VIA_CACHE + url[len("https://") :]
def _per_host_direct(report: AptProxyReportDict) -> set[str]:
hosts = {**report["per_host"], **report["https_per_host"]}
return {h.lower() for h, v in hosts.items() if v.lower() in _NO_PROXY}
def _host_of(source: str) -> str | None:
"""The host of an https source the cache can fetch: port 443, no login in the URL."""
try:
parts = urlsplit(source)
port = parts.port
except ValueError:
return None
if parts.username is not None or port not in (None, 443) or not parts.hostname:
return None
return parts.hostname
def cacheable_https_hosts(report: AptProxyReportDict) -> list[str]:
"""The hosts of the https sources the rewrite would send through the cache, sorted.
None while apt has no http proxy for every host: the cache form goes through it.
Not a host apt fetches ``DIRECT``, nor one the rewrite left on https for its
credentials (``not_cached``), nor one on another port or with a login in the URL.
"""
if not report["proxy"]:
return []
skip = _per_host_direct(report) | {h.lower() for h in report.get("not_cached", [])}
hosts = {_host_of(s) for s in report["https_sources"]}
return sorted(h for h in hosts if h and h.lower() not in skip)
def is_host_name(host: str) -> bool:
"""Whether *host* is a DNS name: letters, digits and hyphens between dots."""
return bool(_HOST.match(host))
def _checked(hosts: list[str]) -> str:
for host in hosts:
if not is_host_name(host):
raise ValueError(f"not a host name: {host!r}")
return " ".join(hosts)
def rewrite_sources_command(
hosts: list[str], proxy_file: str, apt_config: str = "apt-config"
) -> str:
"""The root command that sends the https sources of *hosts* through the cache.
A host apt has credentials for stays on https and is named in *proxy_file*.
*apt_config* is how apt-config is called; a test points apt's paths elsewhere.
:raises ValueError: if one of *hosts* is not a host name.
"""
return _REWRITE.format(
apt_config=apt_config,
hosts=_checked(hosts),
proxy_file=shlex.quote(proxy_file),
key=NOT_CACHED_KEY,
lines=_SOURCE_LINES,
via=VIA_CACHE,
)
def revert_sources_command(apt_config: str = "apt-config") -> str:
"""The root command that turns every source in the cache form back to https."""
return _REVERT.format(apt_config=apt_config, lines=_SOURCE_LINES, via=VIA_CACHE)
__all__ = [
"NOT_CACHED_KEY",
"VIA_CACHE",
"cache_form",
"cacheable_https_hosts",
"is_host_name",
"revert_sources_command",
"rewrite_sources_command",
]