feat(apt): send https sources through the cache in the form it stores (NetOrk/netork#2555)
driver / lint (pull_request) Successful in 15s
driver / standard (pull_request) Successful in 19s
driver / types (pull_request) Successful in 58s
driver / test (3.13) (pull_request) Successful in 1m1s
driver / test (3.12) (pull_request) Successful in 1m8s
driver / lowest (pull_request) Successful in 35s
driver / build (pull_request) Successful in 29s
driver / security (pull_request) Successful in 1m9s
CI / driver (pull_request) Successful in 2m29s
driver / publish (pull_request) Skipped

apt-cacher-ng tunnels a source fetched over https and stores nothing of it. It
fetches http://HTTPS///<host>/<path> over https itself, with the certificate
verified, and stores it, where port 443 is allowed (AllowUserPorts=80 443).

- apt_sources: cacheable_https_hosts(report), and the root commands that rewrite
  the https sources of .list and deb822 .sources files to that form and back.
  A host apt has credentials for in auth.conf stays on https; the rewrite names
  it in the proxy file as Package-Proxy::Not-Cached, which the report reads.
- configure_package_proxy(url): a step "cache" before "https". It rewrites the
  sources, fetches one of them through url, and turns them all back to https
  where url cannot. The "fetch" step proves the proxy with a plain http source.
- configure_package_proxy(""): the sources go back to https before the proxy
  file goes; without the cache apt would look for a host called HTTPS.
This commit is contained in:
christianmanivong committed 2026-11-15 20:07:27 +01:00
1 parent b4929990f1
commit cb0f1cc05c
5 files changed
+808 -18

No files matched your search

+195 -1
View File
@@ -25,6 +25,7 @@ from napalm_device_types.apt_proxy import (
parse_apt_proxy,
same_proxy_url,
)
from napalm_device_types.apt_sources import NOT_CACHED_KEY, VIA_CACHE
from napalm_device_types.fixes import PackageProxyMixin
@@ -302,6 +303,13 @@ _CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/
E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142]
E: Download Failed"""
#: A cache that does not fetch the cache form itself: ``AllowUserPorts`` is 80 alone
#: (apt-cacher-ng 3.7.5, NetOrk/netork#2555).
_PROHIBITED_PORT = """Err:1 http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease
403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143]
E: Failed to fetch http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143]
E: Download Failed"""
def _fetch(output, rc, match=None):
lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)]
@@ -422,9 +430,21 @@ class _Host(AptProxyMixin):
"""
def __init__(
self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0
self,
files=None,
fetch_rc=0,
apt=True,
https_sources=(),
tunnel=True,
direct_rc=0,
cache_https=False,
credentials=(),
):
self.files = dict(files or {})
#: Whether the proxy fetches the cache form itself (AllowUserPorts 443), and
#: the hosts apt keeps credentials for.
self.cache_https = cache_https
self.credentials = set(credentials)
self.fetch_rc = fetch_rc
self.apt = apt
self.https_sources = list(https_sources)
@@ -459,6 +479,12 @@ class _Host(AptProxyMixin):
value, https = self._effective(), self._effective("https")
dump = [f'Acquire::http::Proxy "{value}";'] if value else []
dump += [f'Acquire::https::Proxy "{https}";'] if https else []
dump += [
line
for path, text in self.files.items()
for line in text.splitlines()
if line.startswith(NOT_CACHED_KEY)
]
return _report(
dump=dump,
parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)],
@@ -466,6 +492,8 @@ class _Host(AptProxyMixin):
sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources],
)
self.fetched.append(command)
if re.search(r"download-file '?http://HTTPS///", command):
return _fetch(_FETCHED, 0) if self.cache_https else _fetch(_PROHIBITED_PORT, 100)
if re.search(r"download-file '?https://", command):
return self._fetch_https(command)
return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1)
@@ -482,6 +510,11 @@ class _Host(AptProxyMixin):
def _run_apt_root_command(self, command, done):
self.root.append(command)
if NOT_CACHED_KEY in command:
return self._rewrite(re.search(r"for h in ([^;]+); do", command).group(1).split(), done)
if "#\\1https://#g" in command:
self.https_sources = [s.replace(VIA_CACHE, "https://") for s in self.https_sources]
return {"ok": True, "output": done}
path = command.rsplit(" ", 1)[-1].strip("'")
for m in re.finditer(r"(\d+)\{/[^}]*\}", command):
n = int(m.group(1))
@@ -492,6 +525,19 @@ class _Host(AptProxyMixin):
self.files.pop(command.split()[-1].strip("'"), None)
return {"ok": True, "output": done}
def _rewrite(self, hosts, done):
for host in hosts:
if host in self.credentials:
proxy = self.files.get(f"{_D}00proxy", "")
self.files[f"{_D}00proxy"] = proxy + f'{NOT_CACHED_KEY}:: "{host}";\n'
continue
prefix = f"https://{host}/"
self.https_sources = [
VIA_CACHE + s[len("https://") :] if s.startswith(prefix) else s
for s in self.https_sources
]
return {"ok": True, "output": done}
class TestCheckHttpsSource:
def test_one_source_the_way_apt_fetches_it(self):
@@ -569,8 +615,10 @@ class TestConfigurePackageProxy:
"conflicts",
"verify",
"fetch",
"cache",
"https",
]
assert "no https source" in _step(result, "cache")["output"]
assert "no https source" in result["steps"][-1]["output"]
assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n'
assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0]
@@ -766,6 +814,144 @@ class TestConfigurePackageProxy:
assert host.files == {}
class TestCacheHttpsSources:
"""NetOrk/netork#2555: the cache stores an https source only in its own form."""
def test_a_cache_that_fetches_https_gets_them(self):
host = _Host(https_sources=[_DOCKER], cache_https=True)
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is True
assert host.https_sources == [f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"]
assert _step(result, "cache")["output"] == (
"https sources cached through http://c:3142: download.docker.com"
)
assert any(VIA_CACHE in c for c in host.fetched)
assert "no https source goes through" in _step(result, "https")["output"]
def test_a_cache_that_does_not_turns_them_back(self):
host = _Host(https_sources=[_DOCKER])
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is True
assert host.https_sources == [_DOCKER]
output = _step(result, "cache")["output"]
assert "http://c:3142 does not fetch https sources itself" in output
assert "prohibited port (see AllowUserPorts)" in output
assert "they stay on https, not cached" in output
assert _step(result, "https")["output"] == (
f"fetched {_DOCKER}InRelease through http://c:3142"
)
def test_a_host_with_credentials_stays_on_https_and_is_named(self):
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
host = _Host(
https_sources=[pve, _DOCKER],
cache_https=True,
credentials={"enterprise.proxmox.com"},
)
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is True
assert pve in host.https_sources
output = _step(result, "cache")["output"]
assert "cached through http://c:3142: download.docker.com" in output
assert "left on https, apt has credentials for them: enterprise.proxmox.com" in output
assert host.get_package_proxy_report()["not_cached"] == ["enterprise.proxmox.com"]
def test_only_hosts_with_credentials(self):
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
host = _Host(https_sources=[pve], cache_https=True, credentials={"enterprise.proxmox.com"})
result = host.configure_package_proxy("http://c:3142")
assert _step(result, "cache")["output"] == (
"no https source to cache; "
"left on https, apt has credentials for them: enterprise.proxmox.com"
)
def test_the_https_direct_fallback_keeps_the_hosts_left_on_https(self):
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
host = _Host(https_sources=[pve], tunnel=False, credentials={"enterprise.proxmox.com"})
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is True
assert host.files[f"{_D}00proxy"] == (
'Acquire::http::Proxy "http://c:3142";\n'
'Acquire::https::Proxy "DIRECT";\n'
f'{NOT_CACHED_KEY}:: "enterprise.proxmox.com";\n'
)
def test_a_rewrite_that_fails_stops_the_fix(self):
host = _Host(https_sources=[_DOCKER], cache_https=True)
real = host._run_apt_root_command
def denied(command, done):
if NOT_CACHED_KEY in command:
return {"ok": False, "output": "sed: denied"}
return real(command, done)
host._run_apt_root_command = denied
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is False
assert result["steps"][-1] == {"step": "cache", "ok": False, "output": "sed: denied"}
def test_a_turn_back_that_fails_says_so(self):
host = _Host(https_sources=[_DOCKER])
real = host._run_apt_root_command
def denied(command, done):
if "#\\1https://#g" in command:
return {"ok": False, "output": "sed: denied"}
return real(command, done)
host._run_apt_root_command = denied
result = host.configure_package_proxy("http://c:3142")
assert result["success"] is False
output = result["steps"][-1]["output"]
assert "turning the sources back to https failed: sed: denied" in output
def test_the_proof_of_the_proxy_is_a_plain_http_source(self):
"""A source already in the cache form says nothing about plain http."""
host = _Host(https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"])
host.configure_package_proxy("http://c:3142")
fetch = next(c for c in host.fetched if "InRelease" in c)
assert "deb.debian.org" in fetch
def test_removing_the_proxy_turns_the_sources_back_first(self):
host = _Host(
{f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'},
https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"],
)
result = host.configure_package_proxy("")
assert result["success"] is True
assert [s["step"] for s in result["steps"]] == ["sources", "configure"]
assert host.https_sources == [_DOCKER]
assert host.files == {}
def test_a_turn_back_that_fails_keeps_the_proxy(self):
"""Without the cache apt looks for a host called HTTPS: the proxy stays."""
host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'})
host._run_apt_root_command = lambda command, done: {"ok": False, "output": "denied"}
result = host.configure_package_proxy("")
assert result["success"] is False
assert f"{_D}00proxy" in host.files
class TestWhatTheParserSkips:
def test_a_line_that_is_no_grep_output(self):
assert (
@@ -824,3 +1010,11 @@ def test_the_comment_out_command_for_https():
assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == (
"sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf"
)
def test_a_line_left_on_https_that_is_no_host_name_is_not_written_back():
host = _Host()
host._write_package_proxy("http://c:3142", not_cached=["ok.example", 'x"; Foo "1'])
assert host.files[f"{_D}00proxy"] == (
f'Acquire::http::Proxy "http://c:3142";\n{NOT_CACHED_KEY}:: "ok.example";\n'
)
+318
View File
@@ -0,0 +1,318 @@
"""https sources through the cache: which can be, and the root commands that turn them.
apt-cacher-ng fetches ``http://HTTPS///<host>/<path>`` from ``https://<host>/<path>``
and stores it, where it allows port 443 (``AllowUserPorts=80 443``); a source
fetched over https itself goes through the cache's tunnel and is never stored
(NetOrk/netork#2555). The commands run here for real, on files in a temporary
directory that ``apt-config -o`` points apt's own settings at.
"""
from __future__ import annotations
import shutil
import subprocess
import pytest
from napalm_device_types.apt_proxy import parse_apt_proxy
from napalm_device_types.apt_sources import (
VIA_CACHE,
cache_form,
cacheable_https_hosts,
is_host_name,
revert_sources_command,
rewrite_sources_command,
)
def _report(**over):
report = {
"proxy": "http://c:3142",
"per_host": {},
"auto_detect": None,
"settings": [],
"https_proxy": "http://c:3142",
"https_per_host": {},
"https_settings": [],
"http_sources": ["http://deb.debian.org/debian/dists/trixie/"],
"https_sources": [],
"not_cached": [],
}
report.update(over)
return report
class TestCacheForm:
def test_an_https_url(self):
assert cache_form("https://download.docker.com/linux/ubuntu/dists/noble/") == (
"http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/"
)
def test_the_prefix(self):
assert VIA_CACHE == "http://HTTPS///"
def test_not_an_https_url(self):
with pytest.raises(ValueError):
cache_form("http://deb.debian.org/debian/")
class TestCacheableHttpsHosts:
def test_each_host_once_sorted(self):
report = _report(
https_sources=[
"https://packages.wazuh.com/4.x/apt/dists/stable/",
"https://download.docker.com/linux/ubuntu/dists/noble/",
"https://download.docker.com/linux/ubuntu/dists/noble-test/",
]
)
assert cacheable_https_hosts(report) == ["download.docker.com", "packages.wazuh.com"]
def test_none_without_an_http_proxy(self):
# The cache form goes through the http proxy: without one it goes nowhere.
report = _report(proxy=None, https_sources=["https://h.example/dists/x/"])
assert cacheable_https_hosts(report) == []
def test_a_host_apt_fetches_direct_stays(self):
report = _report(
per_host={"a.example": "DIRECT"},
https_per_host={"B.example": "direct"},
https_sources=[
"https://a.example/dists/x/",
"https://b.example/dists/x/",
"https://c.example/dists/x/",
],
)
assert cacheable_https_hosts(report) == ["c.example"]
def test_a_host_with_credentials_stays(self):
report = _report(
https_sources=["https://enterprise.proxmox.com/debian/pve/dists/trixie/"],
not_cached=["enterprise.proxmox.com"],
)
assert cacheable_https_hosts(report) == []
def test_another_port_or_a_login_in_the_url_stays(self):
# The cache allows port 443 alone, and a login in the URL is a credential too.
report = _report(
https_sources=[
"https://h.example:8443/dists/x/",
"https://u:p@i.example/dists/x/",
"https://j.example:443/dists/x/",
]
)
assert cacheable_https_hosts(report) == ["j.example"]
def test_a_report_from_before_not_cached(self):
report = _report(https_sources=["https://h.example/dists/x/"])
del report["not_cached"]
assert cacheable_https_hosts(report) == ["h.example"]
class TestNotCachedInTheReport:
def test_the_dump_names_them(self):
output = "\n".join(
[
"APROXY_BEGIN",
"[dump]",
'Acquire::http::Proxy "http://c:3142";',
'Package-Proxy::Not-Cached "";',
'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";',
'Package-Proxy::Not-Cached:: "esm.ubuntu.com";',
"[sources]",
"APROXY_END",
]
)
assert parse_apt_proxy(output)["not_cached"] == [
"enterprise.proxmox.com",
"esm.ubuntu.com",
]
def test_none(self):
assert parse_apt_proxy("APROXY_BEGIN\n[dump]\nAPROXY_END")["not_cached"] == []
def test_the_hosts_are_checked():
with pytest.raises(ValueError):
rewrite_sources_command(["h.example; rm -rf /"], "/etc/apt/apt.conf.d/00proxy")
_APT_CONFIG = shutil.which("apt-config")
needs_apt = pytest.mark.skipif(_APT_CONFIG is None, reason="apt-config is not on this host")
@pytest.fixture
def etc(tmp_path):
"""apt's sources, its credentials and the proxy file, under *tmp_path*."""
(tmp_path / "sources.list.d").mkdir()
(tmp_path / "auth.conf.d").mkdir()
(tmp_path / "00proxy").write_text('Acquire::http::Proxy "http://c:3142";\n')
return tmp_path
def _apt_config(etc):
return (
f"apt-config -o Dir::Etc::sourcelist={etc}/sources.list "
f"-o Dir::Etc::sourceparts={etc}/sources.list.d/ "
f"-o Dir::Etc::netrc={etc}/auth.conf -o Dir::Etc::netrcparts={etc}/auth.conf.d/"
)
def _run(command):
done = subprocess.run(["sh", "-c", command], capture_output=True, text=True, check=False)
assert done.returncode == 0, done.stderr
return done.stdout
_DOCKER_LIST = (
"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] "
"https://download.docker.com/linux/ubuntu noble stable\n"
)
_DOCKER_SOURCES = (
"Types: deb\n"
"URIs: https://download.docker.com/linux/ubuntu\n"
"Suites: noble\n"
"Components: stable\n"
"Signed-By: /etc/apt/keyrings/docker.asc\n"
)
@needs_apt
class TestRewrite:
def test_a_one_line_source(self, etc):
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
assert (etc / "sources.list.d" / "docker.list").read_text() == (
"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] "
"http://HTTPS///download.docker.com/linux/ubuntu noble stable\n"
)
def test_a_deb822_source(self, etc):
(etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES)
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
assert (etc / "sources.list.d" / "docker.sources").read_text() == (
_DOCKER_SOURCES.replace("https://", "http://HTTPS///")
)
def test_sources_list_itself_and_only_the_hosts_named(self, etc):
(etc / "sources.list").write_text(
"deb http://deb.debian.org/debian trixie main\n"
"deb https://a.example/debian trixie main\n"
"deb-src https://a.example/debian trixie main\n"
"deb https://b.example/debian trixie main\n"
"# deb https://a.example/old trixie main\n"
)
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
assert (etc / "sources.list").read_text() == (
"deb http://deb.debian.org/debian trixie main\n"
"deb http://HTTPS///a.example/debian trixie main\n"
"deb-src http://HTTPS///a.example/debian trixie main\n"
"deb https://b.example/debian trixie main\n"
"# deb https://a.example/old trixie main\n"
)
def test_a_longer_name_or_another_port_stays(self, etc):
text = (
"deb https://a.example.org/debian trixie main\n"
"deb https://a.example:8443/debian trixie main\n"
)
(etc / "sources.list").write_text(text)
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
assert (etc / "sources.list").read_text() == text
def test_a_host_with_credentials_stays_and_is_named(self, etc):
(etc / "sources.list.d" / "pve.sources").write_text(
"Types: deb\nURIs: https://enterprise.proxmox.com/debian/pve\nSuites: trixie\n"
)
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
(etc / "auth.conf.d" / "pve.conf").write_text(
"machine https://Enterprise.Proxmox.com/debian/pve\nlogin key\npassword secret\n"
)
_run(
rewrite_sources_command(
["enterprise.proxmox.com", "download.docker.com"],
f"{etc}/00proxy",
_apt_config(etc),
)
)
assert "https://enterprise.proxmox.com" in (etc / "sources.list.d/pve.sources").read_text()
assert "HTTPS///download.docker.com" in (etc / "sources.list.d/docker.list").read_text()
assert (etc / "00proxy").read_text() == (
'Acquire::http::Proxy "http://c:3142";\n'
'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";\n'
)
assert "secret" not in (etc / "00proxy").read_text()
def test_credentials_in_auth_conf_on_one_line(self, etc):
(etc / "sources.list").write_text("deb https://esm.ubuntu.com/apps/ubuntu noble main\n")
(etc / "auth.conf").write_text("machine esm.ubuntu.com/apps/ubuntu/ login a password b\n")
_run(rewrite_sources_command(["esm.ubuntu.com"], f"{etc}/00proxy", _apt_config(etc)))
assert "https://esm.ubuntu.com" in (etc / "sources.list").read_text()
assert "esm.ubuntu.com" in (etc / "00proxy").read_text()
def test_a_file_without_the_host_is_left_untouched(self, etc):
other = etc / "sources.list.d" / "other.list"
other.write_text("deb http://deb.debian.org/debian trixie main\n")
before = other.stat().st_ino # sed -i writes a new file; grep first leaves it be
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
assert other.stat().st_ino == before
def test_twice_is_once(self, etc):
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
command = rewrite_sources_command(
["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)
)
_run(command)
_run(command)
assert (etc / "sources.list.d" / "docker.list").read_text().count("HTTPS///") == 1
@needs_apt
class TestRevert:
def test_every_source_goes_back_to_https(self, etc):
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
(etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES)
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
_run(revert_sources_command(_apt_config(etc)))
assert (etc / "sources.list.d" / "docker.list").read_text() == _DOCKER_LIST
assert (etc / "sources.list.d" / "docker.sources").read_text() == _DOCKER_SOURCES
def test_nothing_to_revert(self, etc):
(etc / "sources.list").write_text("deb http://deb.debian.org/debian trixie main\n")
_run(revert_sources_command(_apt_config(etc)))
assert (
etc / "sources.list"
).read_text() == "deb http://deb.debian.org/debian trixie main\n"
def test_the_commands_are_valid_posix_sh():
for command in (
rewrite_sources_command(["a.example"], "/etc/apt/apt.conf.d/00proxy"),
revert_sources_command(),
):
assert subprocess.run(["sh", "-n", "-c", command], check=False).returncode == 0
def test_a_host_name():
assert is_host_name("download.docker.com")
assert not is_host_name('h"; Acquire::x "y')
assert not is_host_name("-h.example")