Merge pull request 'feat(apt): send https sources through the cache in the form it stores (netork#2555)' (#122) from fix/apt-https-cache into main
driver / lint (push) Successful in 13s
driver / standard (push) Successful in 16s
driver / types (push) Successful in 58s
driver / test (3.13) (push) Successful in 1m14s
driver / test (3.12) (push) Successful in 1m21s
driver / lowest (push) Successful in 37s
driver / build (push) Successful in 29s
driver / security (push) Successful in 1m3s
CI / driver (push) Successful in 2m33s
driver / publish (push) Skipped
driver / lint (push) Successful in 13s
driver / standard (push) Successful in 16s
driver / types (push) Successful in 58s
driver / test (3.13) (push) Successful in 1m14s
driver / test (3.12) (push) Successful in 1m21s
driver / lowest (push) Successful in 37s
driver / build (push) Successful in 29s
driver / security (push) Successful in 1m3s
CI / driver (push) Successful in 2m33s
driver / publish (push) Skipped
This commit was merged in pull request #122.
This commit is contained in:
commit
7c0e4fffc5
5 files changed
+808
-18
No files matched your search
@@ -7,6 +7,23 @@ recorded here; their history is the git log.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
- `apt_sources`: apt's https sources through the package cache, in the form apt-cacher-ng
|
||||
fetches over https itself and stores (`http://HTTPS///<host>/…`, with `AllowUserPorts=80
|
||||
443` on the cache). `cacheable_https_hosts(report)` names the hosts this applies to, and
|
||||
`rewrite_sources_command` / `revert_sources_command` are the root commands, for `.list`
|
||||
and deb822 `.sources` files. A host apt keeps credentials for in `auth.conf` stays on
|
||||
https, because apt finds them by the host's name and the cache is shared. The rewrite
|
||||
names such a host in the proxy file as `Package-Proxy::Not-Cached` (NetOrk/netork#2555).
|
||||
- `get_package_proxy_report()` reports those hosts as `not_cached`.
|
||||
|
||||
### Changed
|
||||
- `configure_package_proxy(url)` has a step `cache` before `https`. It rewrites the https
|
||||
sources it can, fetches one of them through *url*, and turns them all back to https
|
||||
where *url* cannot. The `fetch` step proves the proxy with a plain http source only.
|
||||
- `configure_package_proxy("")` turns the sources back to https (step `sources`) before it
|
||||
removes the proxy file. Without the cache, apt would look for a host called `HTTPS`.
|
||||
|
||||
## [4.39.0] - 2026-11-14
|
||||
|
||||
### Added
|
||||
|
||||
@@ -35,8 +35,12 @@ page is the right one, the caller says (``expect``).
|
||||
out every other line that sets a different one, reads the result back from
|
||||
``apt-config``, and fetches the ``InRelease`` of one http source through it. The
|
||||
last step is what proves it: apt reached a mirror through the proxy it is now set
|
||||
to. The source chosen is one apt does not fetch ``DIRECT``. Then it fetches one
|
||||
https source the same way; where the proxy refuses it, https goes ``DIRECT``.
|
||||
to. The source chosen is one apt does not fetch ``DIRECT``. Then the https sources
|
||||
go through the proxy in the form it stores (:mod:`~napalm_device_types.apt_sources`),
|
||||
and one of them is fetched; where the proxy cannot, they go back to https. Last it
|
||||
fetches one https source left the same way; where the proxy refuses it, https goes
|
||||
``DIRECT``. Removing the proxy turns the sources back to https first: without the
|
||||
proxy apt cannot fetch the stored form.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -49,6 +53,14 @@ from urllib.parse import urlsplit
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from napalm_device_types.apt_sources import (
|
||||
NOT_CACHED_KEY,
|
||||
VIA_CACHE,
|
||||
cacheable_https_hosts,
|
||||
is_host_name,
|
||||
revert_sources_command,
|
||||
rewrite_sources_command,
|
||||
)
|
||||
from napalm_device_types.fixes import PackageProxyMixin, run_fix_steps
|
||||
from napalm_device_types.models import FixResultDict, FixStepOutcomeDict
|
||||
from napalm_device_types.terminal import strip_terminal_codes
|
||||
@@ -87,6 +99,9 @@ class AptProxyReportDict(TypedDict):
|
||||
#: The base URIs of the host's sources over http and over https.
|
||||
http_sources: list[str]
|
||||
https_sources: list[str]
|
||||
#: The hosts the cache rewrite left on https, for their credentials
|
||||
#: (:data:`~napalm_device_types.apt_sources.NOT_CACHED_KEY`).
|
||||
not_cached: list[str]
|
||||
|
||||
|
||||
class ProxyFetchDict(TypedDict):
|
||||
@@ -123,7 +138,8 @@ PROXY_URL = re.compile(r"^https?://[A-Za-z0-9.\-\[\]:@%_~/]+$")
|
||||
#: them early.
|
||||
APT_PROXY_COMMAND = (
|
||||
"( export LC_ALL=C; printf '%s%s\\n' APROXY_ BEGIN; "
|
||||
"echo '[dump]'; apt-config dump 2>/dev/null | grep -i -E '^Acquire::https?::Proxy'; "
|
||||
"echo '[dump]'; apt-config dump 2>/dev/null "
|
||||
"| grep -i -E '^(Acquire::https?::Proxy|Package-Proxy::)'; "
|
||||
'eval "$(apt-config shell ap_parts Dir::Etc::parts/d ap_main Dir::Etc::main/f 2>/dev/null)"; '
|
||||
"echo '[parts]'; for f in \"${ap_parts:-/etc/apt/apt.conf.d/}\"*; do "
|
||||
'[ -f "$f" ] && grep -Hin proxy "$f"; done; '
|
||||
@@ -133,6 +149,7 @@ APT_PROXY_COMMAND = (
|
||||
"printf '%s%s\\n' APROXY_ END ) 2>/dev/null | cat"
|
||||
)
|
||||
|
||||
_NOT_CACHED = re.compile(rf'^{re.escape(NOT_CACHED_KEY)}::\s+"([^"]+)"', re.I)
|
||||
_DUMP = re.compile(r'^Acquire::(https?)::Proxy(::\S+|-Auto-Detect)?\s+"([^"]*)"', re.I)
|
||||
_GREP = re.compile(r"^(/[^:]+):(\d+):(.*)$")
|
||||
_FLAT = re.compile(r'^\s*Acquire::http::Proxy\s+"([^"]*)"\s*;', re.I)
|
||||
@@ -293,6 +310,7 @@ def parse_apt_proxy(output: str) -> AptProxyReportDict:
|
||||
"https_settings": _settings(lines, _https_setting),
|
||||
"http_sources": _of_scheme(sources, "http"),
|
||||
"https_sources": _of_scheme(sources, "https"),
|
||||
"not_cached": [m.group(1) for m in map(_NOT_CACHED.match, sections.get("dump", [])) if m],
|
||||
}
|
||||
|
||||
|
||||
@@ -498,16 +516,20 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
def configure_package_proxy(self, url: str) -> FixResultDict:
|
||||
"""apt fetches through *url*, and is shown to; ``""`` removes :attr:`APT_PROXY_FILE`.
|
||||
|
||||
https goes through *url* too, unless *url* refuses it: then ``DIRECT``.
|
||||
The https sources go through *url* in the form it stores, where it can; the
|
||||
rest goes through *url* too, unless *url* refuses it: then ``DIRECT``.
|
||||
"""
|
||||
if not url:
|
||||
return run_fix_steps([("configure", self._remove_package_proxy)])
|
||||
return run_fix_steps(
|
||||
[("sources", self._revert_sources), ("configure", self._remove_package_proxy)]
|
||||
)
|
||||
return run_fix_steps(
|
||||
[
|
||||
("configure", lambda: self._write_package_proxy(url)),
|
||||
("conflicts", lambda: self._comment_out_other_proxies(url)),
|
||||
("verify", lambda: self._verify_package_proxy(url)),
|
||||
("fetch", lambda: self._fetch_through_package_proxy(url)),
|
||||
("cache", lambda: self._cache_https_sources(url)),
|
||||
("https", lambda: self._https_through_package_proxy(url)),
|
||||
]
|
||||
)
|
||||
@@ -515,6 +537,11 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
def _not_apt(self) -> FixStepOutcomeDict:
|
||||
return {"ok": False, "output": "not an apt host"}
|
||||
|
||||
def _revert_sources(self) -> FixStepOutcomeDict:
|
||||
if not self._is_apt_host():
|
||||
return self._not_apt()
|
||||
return self._run_apt_root_command(revert_sources_command(), "sources back on https")
|
||||
|
||||
def _remove_package_proxy(self) -> FixStepOutcomeDict:
|
||||
if not self._is_apt_host():
|
||||
return self._not_apt()
|
||||
@@ -522,7 +549,9 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
f"rm -f {shlex.quote(self.APT_PROXY_FILE)}", "proxy removed"
|
||||
)
|
||||
|
||||
def _write_package_proxy(self, url: str, https_direct: bool = False) -> FixStepOutcomeDict:
|
||||
def _write_package_proxy(
|
||||
self, url: str, https_direct: bool = False, not_cached: list[str] | None = None
|
||||
) -> FixStepOutcomeDict:
|
||||
if not self._is_apt_host():
|
||||
return self._not_apt()
|
||||
if not PROXY_URL.match(url):
|
||||
@@ -530,6 +559,10 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
content = f'Acquire::http::Proxy "{url}";\n'
|
||||
if https_direct:
|
||||
content += 'Acquire::https::Proxy "DIRECT";\n'
|
||||
# Host names the report read from this file; one a line could not carry is left out.
|
||||
content += "".join(
|
||||
f'{NOT_CACHED_KEY}:: "{host}";\n' for host in not_cached or [] if is_host_name(host)
|
||||
)
|
||||
return self._write_apt_file(self.APT_PROXY_FILE, content)
|
||||
|
||||
def _report(self) -> AptProxyReportDict:
|
||||
@@ -566,16 +599,7 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
}
|
||||
|
||||
def _fetch_through_package_proxy(self, url: str) -> FixStepOutcomeDict:
|
||||
report = self._report()
|
||||
direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"}
|
||||
source = next(
|
||||
(
|
||||
s
|
||||
for s in report["http_sources"]
|
||||
if "/dists/" in s and (urlsplit(s).hostname or "").lower() not in direct
|
||||
),
|
||||
None,
|
||||
)
|
||||
source = _plain_http_source(self._report())
|
||||
if source is None:
|
||||
return {"ok": True, "output": "no http source to fetch through the proxy; not tried"}
|
||||
target = source.rstrip("/") + "/InRelease"
|
||||
@@ -584,6 +608,49 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
return {"ok": True, "output": f"fetched {target} through {url}"}
|
||||
return {"ok": False, "output": f"{target}: {fetched['error']}"}
|
||||
|
||||
def _cache_https_sources(self, url: str) -> FixStepOutcomeDict:
|
||||
"""The https sources through *url* in the form it stores; back to https where it cannot."""
|
||||
hosts = cacheable_https_hosts(self._report())
|
||||
if not hosts:
|
||||
return {"ok": True, "output": "no https source to cache"}
|
||||
rewritten = self._run_apt_root_command(
|
||||
rewrite_sources_command(hosts, self.APT_PROXY_FILE), "sources rewritten"
|
||||
)
|
||||
if not rewritten["ok"]:
|
||||
return rewritten
|
||||
return self._fetch_cached(url, hosts, self._report())
|
||||
|
||||
def _fetch_cached(
|
||||
self, url: str, hosts: list[str], report: AptProxyReportDict
|
||||
) -> FixStepOutcomeDict:
|
||||
"""One rewritten source through *url*: it stays where *url* fetched it."""
|
||||
left = sorted(set(hosts) & set(report["not_cached"]))
|
||||
named = f"; left on https, apt has credentials for them: {', '.join(left)}" if left else ""
|
||||
target = next(
|
||||
(s.rstrip("/") + "/InRelease" for s in report["http_sources"] if _cached(s)), None
|
||||
)
|
||||
if target is None:
|
||||
return {"ok": True, "output": f"no https source to cache{named}"}
|
||||
fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(target)))
|
||||
if fetched["reachable"] and not fetched["error"]:
|
||||
cached = ", ".join(h for h in hosts if h not in left)
|
||||
return {"ok": True, "output": f"https sources cached through {url}: {cached}{named}"}
|
||||
return self._back_to_https(url, f"{target}: {fetched['error']}", named)
|
||||
|
||||
def _back_to_https(self, url: str, why: str, named: str) -> FixStepOutcomeDict:
|
||||
"""*url* did not fetch the stored form: every source goes back to https."""
|
||||
back = self._run_apt_root_command(revert_sources_command(), "sources back on https")
|
||||
if not back["ok"]:
|
||||
return {
|
||||
"ok": False,
|
||||
"output": f"{why}; turning the sources back to https failed: {back['output']}",
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"output": f"{url} does not fetch https sources itself ({why}); "
|
||||
f"they stay on https, not cached{named}",
|
||||
}
|
||||
|
||||
def _https_through_package_proxy(self, url: str) -> FixStepOutcomeDict:
|
||||
"""An https source through *url*; where *url* refuses it, https goes ``DIRECT``."""
|
||||
found = _https_source(self._report(), lambda proxy: same_proxy_url(proxy, url))
|
||||
@@ -609,7 +676,9 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
|
||||
def _https_direct(self, url: str) -> FixStepOutcomeDict:
|
||||
"""Write https ``DIRECT`` beside *url*, and comment out the later lines to *url*."""
|
||||
written = self._write_package_proxy(url, https_direct=True)
|
||||
written = self._write_package_proxy(
|
||||
url, https_direct=True, not_cached=self._report()["not_cached"]
|
||||
)
|
||||
if not written["ok"]:
|
||||
return written
|
||||
later = self._later_https_lines(self._report(), url)
|
||||
@@ -635,6 +704,26 @@ class AptProxyMixin(PackageProxyMixin):
|
||||
]
|
||||
|
||||
|
||||
def _plain_http_source(report: AptProxyReportDict) -> str | None:
|
||||
"""A suite's http source the proxy takes, not in the cache's https form: its proof."""
|
||||
direct = {h.lower() for h, v in report["per_host"].items() if v.lower() == "direct"}
|
||||
return next(
|
||||
(
|
||||
s
|
||||
for s in report["http_sources"]
|
||||
if "/dists/" in s
|
||||
and not s.startswith(VIA_CACHE)
|
||||
and (urlsplit(s).hostname or "").lower() not in direct
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _cached(source: str) -> bool:
|
||||
"""A source of a suite in the form the cache stores."""
|
||||
return source.startswith(VIA_CACHE) and "/dists/" in source
|
||||
|
||||
|
||||
def _https_source(
|
||||
report: AptProxyReportDict, through: Callable[[str | None], bool]
|
||||
) -> tuple[str, str] | None:
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
"""apt's https sources through the package cache, so that the cache stores them.
|
||||
|
||||
A source fetched over https goes through apt-cacher-ng's tunnel, if at all: the
|
||||
TLS runs end to end and nothing of it is stored. The cache fetches
|
||||
``http://HTTPS///<host>/<path>`` from ``https://<host>/<path>`` itself, with the
|
||||
certificate verified, and stores it -- where it allows port 443
|
||||
(``AllowUserPorts=80 443``) (NetOrk/netork#2555). apt fetches that form through
|
||||
its http proxy, so it works only while the cache is that proxy: without one apt
|
||||
looks for a host called ``HTTPS``.
|
||||
|
||||
**Which hosts** (:func:`cacheable_https_hosts`): the hosts of the https sources,
|
||||
on port 443, while apt has an http proxy for every host and none of its own for
|
||||
that one. Not a host with a login in the URL, nor one whose credentials apt keeps
|
||||
in ``auth.conf``: apt finds those by the host's name, which the rewrite changes,
|
||||
and paid content does not belong in a cache other hosts share. Only root reads
|
||||
``auth.conf``, so the rewrite judges that, leaves such a host on https and names
|
||||
it in the proxy file as ``Package-Proxy::Not-Cached``, which apt ignores and
|
||||
``apt-config dump`` shows to the report.
|
||||
|
||||
**The rewrite** (:func:`rewrite_sources_command`) turns ``https://<host>`` into
|
||||
the cache form on the lines that name sources: ``deb`` and ``deb-src`` in a
|
||||
``.list`` file, ``URIs:`` in a deb822 ``.sources`` file. A file without the host
|
||||
is not touched. :func:`revert_sources_command` turns every source back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shlex
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - the report's type, without an import cycle
|
||||
from napalm_device_types.apt_proxy import AptProxyReportDict
|
||||
|
||||
#: The prefix apt-cacher-ng fetches over https.
|
||||
VIA_CACHE = "http://HTTPS///"
|
||||
|
||||
#: The key in the proxy file that names the hosts left on https for their credentials.
|
||||
NOT_CACHED_KEY = "Package-Proxy::Not-Cached"
|
||||
|
||||
_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?"
|
||||
_HOST = re.compile(rf"^{_LABEL}(?:\.{_LABEL})*$")
|
||||
_NO_PROXY = frozenset({"", "direct"})
|
||||
|
||||
#: Where apt's sources and credentials are, as shell variables; apt's defaults
|
||||
#: when apt-config cannot say.
|
||||
_LOCATE = (
|
||||
'eval "$({apt_config} shell sl Dir::Etc::sourcelist/f sp Dir::Etc::sourceparts/d '
|
||||
'nr Dir::Etc::netrc/f np Dir::Etc::netrcparts/d 2>/dev/null)"; '
|
||||
'sl="${{sl:-/etc/apt/sources.list}}"; sp="${{sp:-/etc/apt/sources.list.d/}}"; '
|
||||
'nr="${{nr:-/etc/apt/auth.conf}}"; np="${{np:-/etc/apt/auth.conf.d/}}"; '
|
||||
)
|
||||
|
||||
#: The lines that name sources: ``deb``/``deb-src`` and deb822's ``URIs:``.
|
||||
_SOURCE_LINES = "/^[[:space:]]*(deb|deb-src|[Uu][Rr][Ii][Ss]:)/"
|
||||
|
||||
#: The hosts apt has credentials for: each word after ``machine``, without its
|
||||
#: scheme, port and path, in lower case.
|
||||
_CREDENTIAL_HOSTS = (
|
||||
'creds=$(cat "$nr" "$np"* 2>/dev/null | tr -s " \\t\\r" "\\n" '
|
||||
"| awk 'p {{ print; p = 0; next }} $0 == \"machine\" {{ p = 1 }}' "
|
||||
"| sed -e 's|^[A-Za-z+]*://||' -e 's|[:/].*||' | tr A-Z a-z); "
|
||||
)
|
||||
|
||||
_REWRITE = (
|
||||
"( " + _LOCATE + _CREDENTIAL_HOSTS + "for h in {hosts}; do "
|
||||
'l=$(printf "%s" "$h" | tr A-Z a-z); '
|
||||
'if printf "%s\\n" "$creds" | grep -qxF -- "$l"; then '
|
||||
'grep -qF -- "\\"$h\\"" {proxy_file} 2>/dev/null || '
|
||||
'printf \'{key}:: "%s";\\n\' "$h" >> {proxy_file} || exit 1; continue; fi; '
|
||||
"e=$(printf '%s' \"$h\" | sed 's/[.]/[.]/g'); "
|
||||
'for f in "$sl" "$sp"*.list "$sp"*.sources; do '
|
||||
'[ -f "$f" ] && grep -qF -- "https://$h" "$f" || continue; '
|
||||
'sed -i -E "{lines} s#([[:space:]:])https://$e([/[:space:]]|\\$)#\\1{via}$h\\2#g" "$f" '
|
||||
"|| exit 1; done; done )"
|
||||
)
|
||||
|
||||
_REVERT = (
|
||||
"( " + _LOCATE + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do '
|
||||
'[ -f "$f" ] && grep -qF -- "{via}" "$f" || continue; '
|
||||
"sed -i -E '{lines} s#([[:space:]:]){via}#\\1https://#g' \"$f\" || exit 1; done )"
|
||||
)
|
||||
|
||||
|
||||
def cache_form(url: str) -> str:
|
||||
"""*url*, an https URL, in the form apt-cacher-ng fetches over https and stores.
|
||||
|
||||
:raises ValueError: if *url* is not an https URL.
|
||||
"""
|
||||
if not url.startswith("https://"):
|
||||
raise ValueError(f"not an https URL: {url!r}")
|
||||
return VIA_CACHE + url[len("https://") :]
|
||||
|
||||
|
||||
def _per_host_direct(report: AptProxyReportDict) -> set[str]:
|
||||
hosts = {**report["per_host"], **report["https_per_host"]}
|
||||
return {h.lower() for h, v in hosts.items() if v.lower() in _NO_PROXY}
|
||||
|
||||
|
||||
def _host_of(source: str) -> str | None:
|
||||
"""The host of an https source the cache can fetch: port 443, no login in the URL."""
|
||||
try:
|
||||
parts = urlsplit(source)
|
||||
port = parts.port
|
||||
except ValueError:
|
||||
return None
|
||||
if parts.username is not None or port not in (None, 443) or not parts.hostname:
|
||||
return None
|
||||
return parts.hostname
|
||||
|
||||
|
||||
def cacheable_https_hosts(report: AptProxyReportDict) -> list[str]:
|
||||
"""The hosts of the https sources the rewrite would send through the cache, sorted.
|
||||
|
||||
None while apt has no http proxy for every host: the cache form goes through it.
|
||||
Not a host apt fetches ``DIRECT``, nor one the rewrite left on https for its
|
||||
credentials (``not_cached``), nor one on another port or with a login in the URL.
|
||||
"""
|
||||
if not report["proxy"]:
|
||||
return []
|
||||
skip = _per_host_direct(report) | {h.lower() for h in report.get("not_cached", [])}
|
||||
hosts = {_host_of(s) for s in report["https_sources"]}
|
||||
return sorted(h for h in hosts if h and h.lower() not in skip)
|
||||
|
||||
|
||||
def is_host_name(host: str) -> bool:
|
||||
"""Whether *host* is a DNS name: letters, digits and hyphens between dots."""
|
||||
return bool(_HOST.match(host))
|
||||
|
||||
|
||||
def _checked(hosts: list[str]) -> str:
|
||||
for host in hosts:
|
||||
if not is_host_name(host):
|
||||
raise ValueError(f"not a host name: {host!r}")
|
||||
return " ".join(hosts)
|
||||
|
||||
|
||||
def rewrite_sources_command(
|
||||
hosts: list[str], proxy_file: str, apt_config: str = "apt-config"
|
||||
) -> str:
|
||||
"""The root command that sends the https sources of *hosts* through the cache.
|
||||
|
||||
A host apt has credentials for stays on https and is named in *proxy_file*.
|
||||
*apt_config* is how apt-config is called; a test points apt's paths elsewhere.
|
||||
|
||||
:raises ValueError: if one of *hosts* is not a host name.
|
||||
"""
|
||||
return _REWRITE.format(
|
||||
apt_config=apt_config,
|
||||
hosts=_checked(hosts),
|
||||
proxy_file=shlex.quote(proxy_file),
|
||||
key=NOT_CACHED_KEY,
|
||||
lines=_SOURCE_LINES,
|
||||
via=VIA_CACHE,
|
||||
)
|
||||
|
||||
|
||||
def revert_sources_command(apt_config: str = "apt-config") -> str:
|
||||
"""The root command that turns every source in the cache form back to https."""
|
||||
return _REVERT.format(apt_config=apt_config, lines=_SOURCE_LINES, via=VIA_CACHE)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NOT_CACHED_KEY",
|
||||
"VIA_CACHE",
|
||||
"cache_form",
|
||||
"cacheable_https_hosts",
|
||||
"is_host_name",
|
||||
"revert_sources_command",
|
||||
"rewrite_sources_command",
|
||||
]
|
||||
+195
-1
@@ -25,6 +25,7 @@ from napalm_device_types.apt_proxy import (
|
||||
parse_apt_proxy,
|
||||
same_proxy_url,
|
||||
)
|
||||
from napalm_device_types.apt_sources import NOT_CACHED_KEY, VIA_CACHE
|
||||
from napalm_device_types.fixes import PackageProxyMixin
|
||||
|
||||
|
||||
@@ -302,6 +303,13 @@ _CONNECT_DENIED = """Err:1 https://download.docker.com/linux/ubuntu/dists/noble/
|
||||
E: Failed to fetch https://download.docker.com/linux/ubuntu/dists/noble/InRelease Invalid response from proxy: HTTP/1.0 403 CONNECT denied (ask the admin to allow HTTPS tunnels) [IP: 10.40.100.10 3142]
|
||||
E: Download Failed"""
|
||||
|
||||
#: A cache that does not fetch the cache form itself: ``AllowUserPorts`` is 80 alone
|
||||
#: (apt-cacher-ng 3.7.5, NetOrk/netork#2555).
|
||||
_PROHIBITED_PORT = """Err:1 http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease
|
||||
403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143]
|
||||
E: Failed to fetch http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease 403 Configuration error (confusing proxy mode) or prohibited port (see AllowUserPorts) [IP: 127.0.0.1 39143]
|
||||
E: Download Failed"""
|
||||
|
||||
|
||||
def _fetch(output, rc, match=None):
|
||||
lines = ["APROXY_BEGIN", "[fetch]", *output.splitlines(), "[rc]", str(rc)]
|
||||
@@ -422,9 +430,21 @@ class _Host(AptProxyMixin):
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, files=None, fetch_rc=0, apt=True, https_sources=(), tunnel=True, direct_rc=0
|
||||
self,
|
||||
files=None,
|
||||
fetch_rc=0,
|
||||
apt=True,
|
||||
https_sources=(),
|
||||
tunnel=True,
|
||||
direct_rc=0,
|
||||
cache_https=False,
|
||||
credentials=(),
|
||||
):
|
||||
self.files = dict(files or {})
|
||||
#: Whether the proxy fetches the cache form itself (AllowUserPorts 443), and
|
||||
#: the hosts apt keeps credentials for.
|
||||
self.cache_https = cache_https
|
||||
self.credentials = set(credentials)
|
||||
self.fetch_rc = fetch_rc
|
||||
self.apt = apt
|
||||
self.https_sources = list(https_sources)
|
||||
@@ -459,6 +479,12 @@ class _Host(AptProxyMixin):
|
||||
value, https = self._effective(), self._effective("https")
|
||||
dump = [f'Acquire::http::Proxy "{value}";'] if value else []
|
||||
dump += [f'Acquire::https::Proxy "{https}";'] if https else []
|
||||
dump += [
|
||||
line
|
||||
for path, text in self.files.items()
|
||||
for line in text.splitlines()
|
||||
if line.startswith(NOT_CACHED_KEY)
|
||||
]
|
||||
return _report(
|
||||
dump=dump,
|
||||
parts=[f"{p}:{n}:{t}" for p, n, t in self._settings() if p.startswith(_D)],
|
||||
@@ -466,6 +492,8 @@ class _Host(AptProxyMixin):
|
||||
sources=["http://deb.debian.org/debian/dists/trixie/", *self.https_sources],
|
||||
)
|
||||
self.fetched.append(command)
|
||||
if re.search(r"download-file '?http://HTTPS///", command):
|
||||
return _fetch(_FETCHED, 0) if self.cache_https else _fetch(_PROHIBITED_PORT, 100)
|
||||
if re.search(r"download-file '?https://", command):
|
||||
return self._fetch_https(command)
|
||||
return _fetch(_FETCHED if self.fetch_rc == 0 else _REFUSED, self.fetch_rc, match=1)
|
||||
@@ -482,6 +510,11 @@ class _Host(AptProxyMixin):
|
||||
|
||||
def _run_apt_root_command(self, command, done):
|
||||
self.root.append(command)
|
||||
if NOT_CACHED_KEY in command:
|
||||
return self._rewrite(re.search(r"for h in ([^;]+); do", command).group(1).split(), done)
|
||||
if "#\\1https://#g" in command:
|
||||
self.https_sources = [s.replace(VIA_CACHE, "https://") for s in self.https_sources]
|
||||
return {"ok": True, "output": done}
|
||||
path = command.rsplit(" ", 1)[-1].strip("'")
|
||||
for m in re.finditer(r"(\d+)\{/[^}]*\}", command):
|
||||
n = int(m.group(1))
|
||||
@@ -492,6 +525,19 @@ class _Host(AptProxyMixin):
|
||||
self.files.pop(command.split()[-1].strip("'"), None)
|
||||
return {"ok": True, "output": done}
|
||||
|
||||
def _rewrite(self, hosts, done):
|
||||
for host in hosts:
|
||||
if host in self.credentials:
|
||||
proxy = self.files.get(f"{_D}00proxy", "")
|
||||
self.files[f"{_D}00proxy"] = proxy + f'{NOT_CACHED_KEY}:: "{host}";\n'
|
||||
continue
|
||||
prefix = f"https://{host}/"
|
||||
self.https_sources = [
|
||||
VIA_CACHE + s[len("https://") :] if s.startswith(prefix) else s
|
||||
for s in self.https_sources
|
||||
]
|
||||
return {"ok": True, "output": done}
|
||||
|
||||
|
||||
class TestCheckHttpsSource:
|
||||
def test_one_source_the_way_apt_fetches_it(self):
|
||||
@@ -569,8 +615,10 @@ class TestConfigurePackageProxy:
|
||||
"conflicts",
|
||||
"verify",
|
||||
"fetch",
|
||||
"cache",
|
||||
"https",
|
||||
]
|
||||
assert "no https source" in _step(result, "cache")["output"]
|
||||
assert "no https source" in result["steps"][-1]["output"]
|
||||
assert host.files[f"{_D}00proxy"] == 'Acquire::http::Proxy "http://c:3142";\n'
|
||||
assert "InRelease" in host.fetched[0] and "DIRECT" not in host.fetched[0]
|
||||
@@ -766,6 +814,144 @@ class TestConfigurePackageProxy:
|
||||
assert host.files == {}
|
||||
|
||||
|
||||
class TestCacheHttpsSources:
|
||||
"""NetOrk/netork#2555: the cache stores an https source only in its own form."""
|
||||
|
||||
def test_a_cache_that_fetches_https_gets_them(self):
|
||||
host = _Host(https_sources=[_DOCKER], cache_https=True)
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is True
|
||||
assert host.https_sources == [f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"]
|
||||
assert _step(result, "cache")["output"] == (
|
||||
"https sources cached through http://c:3142: download.docker.com"
|
||||
)
|
||||
assert any(VIA_CACHE in c for c in host.fetched)
|
||||
assert "no https source goes through" in _step(result, "https")["output"]
|
||||
|
||||
def test_a_cache_that_does_not_turns_them_back(self):
|
||||
host = _Host(https_sources=[_DOCKER])
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is True
|
||||
assert host.https_sources == [_DOCKER]
|
||||
output = _step(result, "cache")["output"]
|
||||
assert "http://c:3142 does not fetch https sources itself" in output
|
||||
assert "prohibited port (see AllowUserPorts)" in output
|
||||
assert "they stay on https, not cached" in output
|
||||
assert _step(result, "https")["output"] == (
|
||||
f"fetched {_DOCKER}InRelease through http://c:3142"
|
||||
)
|
||||
|
||||
def test_a_host_with_credentials_stays_on_https_and_is_named(self):
|
||||
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
|
||||
host = _Host(
|
||||
https_sources=[pve, _DOCKER],
|
||||
cache_https=True,
|
||||
credentials={"enterprise.proxmox.com"},
|
||||
)
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is True
|
||||
assert pve in host.https_sources
|
||||
output = _step(result, "cache")["output"]
|
||||
assert "cached through http://c:3142: download.docker.com" in output
|
||||
assert "left on https, apt has credentials for them: enterprise.proxmox.com" in output
|
||||
assert host.get_package_proxy_report()["not_cached"] == ["enterprise.proxmox.com"]
|
||||
|
||||
def test_only_hosts_with_credentials(self):
|
||||
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
|
||||
host = _Host(https_sources=[pve], cache_https=True, credentials={"enterprise.proxmox.com"})
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert _step(result, "cache")["output"] == (
|
||||
"no https source to cache; "
|
||||
"left on https, apt has credentials for them: enterprise.proxmox.com"
|
||||
)
|
||||
|
||||
def test_the_https_direct_fallback_keeps_the_hosts_left_on_https(self):
|
||||
pve = "https://enterprise.proxmox.com/debian/pve/dists/trixie/"
|
||||
host = _Host(https_sources=[pve], tunnel=False, credentials={"enterprise.proxmox.com"})
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is True
|
||||
assert host.files[f"{_D}00proxy"] == (
|
||||
'Acquire::http::Proxy "http://c:3142";\n'
|
||||
'Acquire::https::Proxy "DIRECT";\n'
|
||||
f'{NOT_CACHED_KEY}:: "enterprise.proxmox.com";\n'
|
||||
)
|
||||
|
||||
def test_a_rewrite_that_fails_stops_the_fix(self):
|
||||
host = _Host(https_sources=[_DOCKER], cache_https=True)
|
||||
real = host._run_apt_root_command
|
||||
|
||||
def denied(command, done):
|
||||
if NOT_CACHED_KEY in command:
|
||||
return {"ok": False, "output": "sed: denied"}
|
||||
return real(command, done)
|
||||
|
||||
host._run_apt_root_command = denied
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is False
|
||||
assert result["steps"][-1] == {"step": "cache", "ok": False, "output": "sed: denied"}
|
||||
|
||||
def test_a_turn_back_that_fails_says_so(self):
|
||||
host = _Host(https_sources=[_DOCKER])
|
||||
real = host._run_apt_root_command
|
||||
|
||||
def denied(command, done):
|
||||
if "#\\1https://#g" in command:
|
||||
return {"ok": False, "output": "sed: denied"}
|
||||
return real(command, done)
|
||||
|
||||
host._run_apt_root_command = denied
|
||||
|
||||
result = host.configure_package_proxy("http://c:3142")
|
||||
|
||||
assert result["success"] is False
|
||||
output = result["steps"][-1]["output"]
|
||||
assert "turning the sources back to https failed: sed: denied" in output
|
||||
|
||||
def test_the_proof_of_the_proxy_is_a_plain_http_source(self):
|
||||
"""A source already in the cache form says nothing about plain http."""
|
||||
host = _Host(https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"])
|
||||
|
||||
host.configure_package_proxy("http://c:3142")
|
||||
|
||||
fetch = next(c for c in host.fetched if "InRelease" in c)
|
||||
assert "deb.debian.org" in fetch
|
||||
|
||||
def test_removing_the_proxy_turns_the_sources_back_first(self):
|
||||
host = _Host(
|
||||
{f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'},
|
||||
https_sources=[f"{VIA_CACHE}download.docker.com/linux/ubuntu/dists/noble/"],
|
||||
)
|
||||
|
||||
result = host.configure_package_proxy("")
|
||||
|
||||
assert result["success"] is True
|
||||
assert [s["step"] for s in result["steps"]] == ["sources", "configure"]
|
||||
assert host.https_sources == [_DOCKER]
|
||||
assert host.files == {}
|
||||
|
||||
def test_a_turn_back_that_fails_keeps_the_proxy(self):
|
||||
"""Without the cache apt looks for a host called HTTPS: the proxy stays."""
|
||||
host = _Host({f"{_D}00proxy": 'Acquire::http::Proxy "http://c:3142";\n'})
|
||||
host._run_apt_root_command = lambda command, done: {"ok": False, "output": "denied"}
|
||||
|
||||
result = host.configure_package_proxy("")
|
||||
|
||||
assert result["success"] is False
|
||||
assert f"{_D}00proxy" in host.files
|
||||
|
||||
|
||||
class TestWhatTheParserSkips:
|
||||
def test_a_line_that_is_no_grep_output(self):
|
||||
assert (
|
||||
@@ -824,3 +1010,11 @@ def test_the_comment_out_command_for_https():
|
||||
assert comment_out_command("/etc/apt/apt.conf", [3], scheme="https") == (
|
||||
"sed -i -e '3{/^[[:space:]]*Acquire::https::Proxy[[:space:]]/Is|^|// |}' /etc/apt/apt.conf"
|
||||
)
|
||||
|
||||
|
||||
def test_a_line_left_on_https_that_is_no_host_name_is_not_written_back():
|
||||
host = _Host()
|
||||
host._write_package_proxy("http://c:3142", not_cached=["ok.example", 'x"; Foo "1'])
|
||||
assert host.files[f"{_D}00proxy"] == (
|
||||
f'Acquire::http::Proxy "http://c:3142";\n{NOT_CACHED_KEY}:: "ok.example";\n'
|
||||
)
|
||||
@@ -0,0 +1,318 @@
|
||||
"""https sources through the cache: which can be, and the root commands that turn them.
|
||||
|
||||
apt-cacher-ng fetches ``http://HTTPS///<host>/<path>`` from ``https://<host>/<path>``
|
||||
and stores it, where it allows port 443 (``AllowUserPorts=80 443``); a source
|
||||
fetched over https itself goes through the cache's tunnel and is never stored
|
||||
(NetOrk/netork#2555). The commands run here for real, on files in a temporary
|
||||
directory that ``apt-config -o`` points apt's own settings at.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from napalm_device_types.apt_proxy import parse_apt_proxy
|
||||
from napalm_device_types.apt_sources import (
|
||||
VIA_CACHE,
|
||||
cache_form,
|
||||
cacheable_https_hosts,
|
||||
is_host_name,
|
||||
revert_sources_command,
|
||||
rewrite_sources_command,
|
||||
)
|
||||
|
||||
|
||||
def _report(**over):
|
||||
report = {
|
||||
"proxy": "http://c:3142",
|
||||
"per_host": {},
|
||||
"auto_detect": None,
|
||||
"settings": [],
|
||||
"https_proxy": "http://c:3142",
|
||||
"https_per_host": {},
|
||||
"https_settings": [],
|
||||
"http_sources": ["http://deb.debian.org/debian/dists/trixie/"],
|
||||
"https_sources": [],
|
||||
"not_cached": [],
|
||||
}
|
||||
report.update(over)
|
||||
return report
|
||||
|
||||
|
||||
class TestCacheForm:
|
||||
def test_an_https_url(self):
|
||||
assert cache_form("https://download.docker.com/linux/ubuntu/dists/noble/") == (
|
||||
"http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/"
|
||||
)
|
||||
|
||||
def test_the_prefix(self):
|
||||
assert VIA_CACHE == "http://HTTPS///"
|
||||
|
||||
def test_not_an_https_url(self):
|
||||
with pytest.raises(ValueError):
|
||||
cache_form("http://deb.debian.org/debian/")
|
||||
|
||||
|
||||
class TestCacheableHttpsHosts:
|
||||
def test_each_host_once_sorted(self):
|
||||
report = _report(
|
||||
https_sources=[
|
||||
"https://packages.wazuh.com/4.x/apt/dists/stable/",
|
||||
"https://download.docker.com/linux/ubuntu/dists/noble/",
|
||||
"https://download.docker.com/linux/ubuntu/dists/noble-test/",
|
||||
]
|
||||
)
|
||||
assert cacheable_https_hosts(report) == ["download.docker.com", "packages.wazuh.com"]
|
||||
|
||||
def test_none_without_an_http_proxy(self):
|
||||
# The cache form goes through the http proxy: without one it goes nowhere.
|
||||
report = _report(proxy=None, https_sources=["https://h.example/dists/x/"])
|
||||
assert cacheable_https_hosts(report) == []
|
||||
|
||||
def test_a_host_apt_fetches_direct_stays(self):
|
||||
report = _report(
|
||||
per_host={"a.example": "DIRECT"},
|
||||
https_per_host={"B.example": "direct"},
|
||||
https_sources=[
|
||||
"https://a.example/dists/x/",
|
||||
"https://b.example/dists/x/",
|
||||
"https://c.example/dists/x/",
|
||||
],
|
||||
)
|
||||
assert cacheable_https_hosts(report) == ["c.example"]
|
||||
|
||||
def test_a_host_with_credentials_stays(self):
|
||||
report = _report(
|
||||
https_sources=["https://enterprise.proxmox.com/debian/pve/dists/trixie/"],
|
||||
not_cached=["enterprise.proxmox.com"],
|
||||
)
|
||||
assert cacheable_https_hosts(report) == []
|
||||
|
||||
def test_another_port_or_a_login_in_the_url_stays(self):
|
||||
# The cache allows port 443 alone, and a login in the URL is a credential too.
|
||||
report = _report(
|
||||
https_sources=[
|
||||
"https://h.example:8443/dists/x/",
|
||||
"https://u:p@i.example/dists/x/",
|
||||
"https://j.example:443/dists/x/",
|
||||
]
|
||||
)
|
||||
assert cacheable_https_hosts(report) == ["j.example"]
|
||||
|
||||
def test_a_report_from_before_not_cached(self):
|
||||
report = _report(https_sources=["https://h.example/dists/x/"])
|
||||
del report["not_cached"]
|
||||
assert cacheable_https_hosts(report) == ["h.example"]
|
||||
|
||||
|
||||
class TestNotCachedInTheReport:
|
||||
def test_the_dump_names_them(self):
|
||||
output = "\n".join(
|
||||
[
|
||||
"APROXY_BEGIN",
|
||||
"[dump]",
|
||||
'Acquire::http::Proxy "http://c:3142";',
|
||||
'Package-Proxy::Not-Cached "";',
|
||||
'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";',
|
||||
'Package-Proxy::Not-Cached:: "esm.ubuntu.com";',
|
||||
"[sources]",
|
||||
"APROXY_END",
|
||||
]
|
||||
)
|
||||
assert parse_apt_proxy(output)["not_cached"] == [
|
||||
"enterprise.proxmox.com",
|
||||
"esm.ubuntu.com",
|
||||
]
|
||||
|
||||
def test_none(self):
|
||||
assert parse_apt_proxy("APROXY_BEGIN\n[dump]\nAPROXY_END")["not_cached"] == []
|
||||
|
||||
|
||||
def test_the_hosts_are_checked():
|
||||
with pytest.raises(ValueError):
|
||||
rewrite_sources_command(["h.example; rm -rf /"], "/etc/apt/apt.conf.d/00proxy")
|
||||
|
||||
|
||||
_APT_CONFIG = shutil.which("apt-config")
|
||||
needs_apt = pytest.mark.skipif(_APT_CONFIG is None, reason="apt-config is not on this host")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def etc(tmp_path):
|
||||
"""apt's sources, its credentials and the proxy file, under *tmp_path*."""
|
||||
(tmp_path / "sources.list.d").mkdir()
|
||||
(tmp_path / "auth.conf.d").mkdir()
|
||||
(tmp_path / "00proxy").write_text('Acquire::http::Proxy "http://c:3142";\n')
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _apt_config(etc):
|
||||
return (
|
||||
f"apt-config -o Dir::Etc::sourcelist={etc}/sources.list "
|
||||
f"-o Dir::Etc::sourceparts={etc}/sources.list.d/ "
|
||||
f"-o Dir::Etc::netrc={etc}/auth.conf -o Dir::Etc::netrcparts={etc}/auth.conf.d/"
|
||||
)
|
||||
|
||||
|
||||
def _run(command):
|
||||
done = subprocess.run(["sh", "-c", command], capture_output=True, text=True, check=False)
|
||||
assert done.returncode == 0, done.stderr
|
||||
return done.stdout
|
||||
|
||||
|
||||
_DOCKER_LIST = (
|
||||
"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] "
|
||||
"https://download.docker.com/linux/ubuntu noble stable\n"
|
||||
)
|
||||
_DOCKER_SOURCES = (
|
||||
"Types: deb\n"
|
||||
"URIs: https://download.docker.com/linux/ubuntu\n"
|
||||
"Suites: noble\n"
|
||||
"Components: stable\n"
|
||||
"Signed-By: /etc/apt/keyrings/docker.asc\n"
|
||||
)
|
||||
|
||||
|
||||
@needs_apt
|
||||
class TestRewrite:
|
||||
def test_a_one_line_source(self, etc):
|
||||
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
|
||||
|
||||
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert (etc / "sources.list.d" / "docker.list").read_text() == (
|
||||
"deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] "
|
||||
"http://HTTPS///download.docker.com/linux/ubuntu noble stable\n"
|
||||
)
|
||||
|
||||
def test_a_deb822_source(self, etc):
|
||||
(etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES)
|
||||
|
||||
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert (etc / "sources.list.d" / "docker.sources").read_text() == (
|
||||
_DOCKER_SOURCES.replace("https://", "http://HTTPS///")
|
||||
)
|
||||
|
||||
def test_sources_list_itself_and_only_the_hosts_named(self, etc):
|
||||
(etc / "sources.list").write_text(
|
||||
"deb http://deb.debian.org/debian trixie main\n"
|
||||
"deb https://a.example/debian trixie main\n"
|
||||
"deb-src https://a.example/debian trixie main\n"
|
||||
"deb https://b.example/debian trixie main\n"
|
||||
"# deb https://a.example/old trixie main\n"
|
||||
)
|
||||
|
||||
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert (etc / "sources.list").read_text() == (
|
||||
"deb http://deb.debian.org/debian trixie main\n"
|
||||
"deb http://HTTPS///a.example/debian trixie main\n"
|
||||
"deb-src http://HTTPS///a.example/debian trixie main\n"
|
||||
"deb https://b.example/debian trixie main\n"
|
||||
"# deb https://a.example/old trixie main\n"
|
||||
)
|
||||
|
||||
def test_a_longer_name_or_another_port_stays(self, etc):
|
||||
text = (
|
||||
"deb https://a.example.org/debian trixie main\n"
|
||||
"deb https://a.example:8443/debian trixie main\n"
|
||||
)
|
||||
(etc / "sources.list").write_text(text)
|
||||
|
||||
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert (etc / "sources.list").read_text() == text
|
||||
|
||||
def test_a_host_with_credentials_stays_and_is_named(self, etc):
|
||||
(etc / "sources.list.d" / "pve.sources").write_text(
|
||||
"Types: deb\nURIs: https://enterprise.proxmox.com/debian/pve\nSuites: trixie\n"
|
||||
)
|
||||
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
|
||||
(etc / "auth.conf.d" / "pve.conf").write_text(
|
||||
"machine https://Enterprise.Proxmox.com/debian/pve\nlogin key\npassword secret\n"
|
||||
)
|
||||
|
||||
_run(
|
||||
rewrite_sources_command(
|
||||
["enterprise.proxmox.com", "download.docker.com"],
|
||||
f"{etc}/00proxy",
|
||||
_apt_config(etc),
|
||||
)
|
||||
)
|
||||
|
||||
assert "https://enterprise.proxmox.com" in (etc / "sources.list.d/pve.sources").read_text()
|
||||
assert "HTTPS///download.docker.com" in (etc / "sources.list.d/docker.list").read_text()
|
||||
assert (etc / "00proxy").read_text() == (
|
||||
'Acquire::http::Proxy "http://c:3142";\n'
|
||||
'Package-Proxy::Not-Cached:: "enterprise.proxmox.com";\n'
|
||||
)
|
||||
assert "secret" not in (etc / "00proxy").read_text()
|
||||
|
||||
def test_credentials_in_auth_conf_on_one_line(self, etc):
|
||||
(etc / "sources.list").write_text("deb https://esm.ubuntu.com/apps/ubuntu noble main\n")
|
||||
(etc / "auth.conf").write_text("machine esm.ubuntu.com/apps/ubuntu/ login a password b\n")
|
||||
|
||||
_run(rewrite_sources_command(["esm.ubuntu.com"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert "https://esm.ubuntu.com" in (etc / "sources.list").read_text()
|
||||
assert "esm.ubuntu.com" in (etc / "00proxy").read_text()
|
||||
|
||||
def test_a_file_without_the_host_is_left_untouched(self, etc):
|
||||
other = etc / "sources.list.d" / "other.list"
|
||||
other.write_text("deb http://deb.debian.org/debian trixie main\n")
|
||||
before = other.stat().st_ino # sed -i writes a new file; grep first leaves it be
|
||||
|
||||
_run(rewrite_sources_command(["a.example"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
assert other.stat().st_ino == before
|
||||
|
||||
def test_twice_is_once(self, etc):
|
||||
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
|
||||
command = rewrite_sources_command(
|
||||
["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)
|
||||
)
|
||||
|
||||
_run(command)
|
||||
_run(command)
|
||||
|
||||
assert (etc / "sources.list.d" / "docker.list").read_text().count("HTTPS///") == 1
|
||||
|
||||
|
||||
@needs_apt
|
||||
class TestRevert:
|
||||
def test_every_source_goes_back_to_https(self, etc):
|
||||
(etc / "sources.list.d" / "docker.list").write_text(_DOCKER_LIST)
|
||||
(etc / "sources.list.d" / "docker.sources").write_text(_DOCKER_SOURCES)
|
||||
_run(rewrite_sources_command(["download.docker.com"], f"{etc}/00proxy", _apt_config(etc)))
|
||||
|
||||
_run(revert_sources_command(_apt_config(etc)))
|
||||
|
||||
assert (etc / "sources.list.d" / "docker.list").read_text() == _DOCKER_LIST
|
||||
assert (etc / "sources.list.d" / "docker.sources").read_text() == _DOCKER_SOURCES
|
||||
|
||||
def test_nothing_to_revert(self, etc):
|
||||
(etc / "sources.list").write_text("deb http://deb.debian.org/debian trixie main\n")
|
||||
|
||||
_run(revert_sources_command(_apt_config(etc)))
|
||||
|
||||
assert (
|
||||
etc / "sources.list"
|
||||
).read_text() == "deb http://deb.debian.org/debian trixie main\n"
|
||||
|
||||
|
||||
def test_the_commands_are_valid_posix_sh():
|
||||
for command in (
|
||||
rewrite_sources_command(["a.example"], "/etc/apt/apt.conf.d/00proxy"),
|
||||
revert_sources_command(),
|
||||
):
|
||||
assert subprocess.run(["sh", "-n", "-c", command], check=False).returncode == 0
|
||||
|
||||
|
||||
def test_a_host_name():
|
||||
assert is_host_name("download.docker.com")
|
||||
assert not is_host_name('h"; Acquire::x "y')
|
||||
assert not is_host_name("-h.example")
|
||||
Reference in new issue
Block a user