diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ce0d86..6e30484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ recorded here; their history is the git log. ## [Unreleased] +### Added +- `AptReleaseMixin.get_apt_source_releases()`: the host's release from `/etc/os-release` + (`ID`, `ID_LIKE`, `VERSION_CODENAME`, `UBUNTU_CODENAME`) and every enabled source apt + reads, one-line and deb822, with its file, line, types, URI, suites and components + (NetOrk/netork#2614). `AptProxyMixin` brings it, so every apt driver has it. +- `AptReleaseMixin.set_apt_source_release(path, release, codename)`: every suite of + *release* in one of apt's source files moved to *codename* (`focal-security` → + `noble-security`), once each new suite's `InRelease` or `Release` is fetched with apt's + downloader; nothing is written where the vendor has none. +- `parse_apt_releases`, `rewrite_release`, `suite_release`, `APT_RELEASES_COMMAND`. + ## [4.41.0] - 2026-11-17 ### Added diff --git a/README.md b/README.md index c65b7db..feda9de 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,15 @@ wins (`get_package_proxy`, `get_package_proxy_report` with every line that sets different proxy, reads the result back and fetches one http source's `InRelease` through it. A driver supplies `_run_apt_command`, `_run_apt_root_command` and `_write_apt_file`. +`AptReleaseMixin` (`napalm_device_types.apt_releases`) comes with it: every source apt reads, +with its file, line, URI and suites, and the host's release from `/etc/os-release` +(`get_apt_source_releases`). A release upgrade leaves third-party sources on the old release, +where they stop getting updates once the vendor stops building for it; whether a suite names +another release than the host's is the caller's to judge (`suite_release` gives the word). +`set_apt_source_release(path, release, codename)` moves every suite of *release* in one +source file to *codename*, after fetching each new suite's `InRelease` with apt's downloader: +where the vendor has none, the file stays as it is. + `HostStatusMixin` (`get_host_status`) is mixed in the same way: whether a Linux host needs a reboot to finish an update (`/var/run/reboot-required`, `needs-restarting -r`, or a newer kernel of the running flavour installed), whether it patches itself (unattended-upgrades, diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index 6634702..a7f5ca1 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -78,6 +78,7 @@ The same shape replaces every action in today's `run_device_action`: |---|---| | `fix_snmp` | `configure_snmp_agent(settings)` | | `fix_apt_proxy` | `configure_package_proxy(url)` | +| an apt source on another release than the host (NetOrk/netork#2614) | `set_apt_source_release(path, release, codename)` | | `fix_docker_permissions` | `grant_container_engine_access(user)` | | `install_lldpd`, `install_auc` | `ensure_package(name)` | | the Zabbix agent where no container engine runs (NetOrk/netork#2571) | `configure_zabbix_agent(settings)` | @@ -159,6 +160,7 @@ the driver. | `virtual_ips` | vip-manager, keepalived, awk | | `system_identity` | SMBIOS via sysfs and `dmidecode` | | `package_updates` | apt and dnf output | +| `apt_releases` | apt's source files (one-line and deb822), `/etc/os-release`, `apt-helper` | | `ssh_server` | OpenSSH (`sshd -T`, `sshd -t`, `sshd_config`); which algorithms to drop stays the caller's | | `terminal` | ANSI escape codes in shell output | | `channel`: `run_on_transport`, `open_stream_on_transport`, `ParamikoExecStream` | paramiko exec channels | diff --git a/napalm_device_types/__init__.py b/napalm_device_types/__init__.py index 573b4ba..e700db6 100644 --- a/napalm_device_types/__init__.py +++ b/napalm_device_types/__init__.py @@ -86,6 +86,15 @@ from napalm_device_types.apt_proxy import ( parse_apt_proxy, same_proxy_url, ) +from napalm_device_types.apt_releases import ( + APT_RELEASES_COMMAND, + AptReleaseFixDict, + AptReleaseMixin, + AptReleasesDict, + AptSourceDict, + parse_apt_releases, + suite_release, +) from napalm_device_types.arp import ARP_AGE_UNKNOWN from napalm_device_types.backup_server import BackupServerDriver from napalm_device_types.backups import ( @@ -396,6 +405,13 @@ __all__ = [ "parse_apt_fetch", "parse_apt_proxy", "same_proxy_url", + "APT_RELEASES_COMMAND", + "AptReleaseFixDict", + "AptReleaseMixin", + "AptReleasesDict", + "AptSourceDict", + "parse_apt_releases", + "suite_release", "SnmpAgentMixin", "SnmpAgentSettingsDict", "run_fix_steps", diff --git a/napalm_device_types/apt_proxy.py b/napalm_device_types/apt_proxy.py index 9be8052..3a3a5b5 100644 --- a/napalm_device_types/apt_proxy.py +++ b/napalm_device_types/apt_proxy.py @@ -53,6 +53,7 @@ from urllib.parse import urlsplit from typing_extensions import TypedDict +from napalm_device_types.apt_releases import AptReleaseMixin from napalm_device_types.apt_sources import ( NOT_CACHED_KEY, VIA_CACHE, @@ -427,7 +428,7 @@ def _described(s: AptProxySettingDict) -> str: return f"{s['file']}:{s['line']} ({s['value']}) {what}" -class AptProxyMixin(PackageProxyMixin): +class AptProxyMixin(AptReleaseMixin, PackageProxyMixin): """:class:`~napalm_device_types.fixes.PackageProxyMixin` for a Debian-family host. The template form, like :class:`~napalm_device_types.host_status.HostStatusMixin`: diff --git a/napalm_device_types/apt_releases.py b/napalm_device_types/apt_releases.py new file mode 100644 index 0000000..925bf96 --- /dev/null +++ b/napalm_device_types/apt_releases.py @@ -0,0 +1,486 @@ +"""apt's sources and the release each names, and moving one file's sources to the host's. + +A release upgrade moves the distribution's own sources to the new release and +leaves the third-party ones where they were -- or turns them off, and the person +who turns them back on keeps the old name. A source on the old release still +answers, so apt says nothing; its packages simply stop getting updates once the +vendor stops building for that release. Docker's last build for Ubuntu 20.04 +(``focal``) is 28.1.1, and a noble host on ``focal`` never sees a newer one +(NetOrk/netork#2614). + +**What is read** (:meth:`AptReleaseMixin.get_apt_source_releases`): the host's +``/etc/os-release`` (``ID``, ``ID_LIKE``, ``VERSION_CODENAME``, +``UBUNTU_CODENAME``) and every source apt reads -- ``sources.list`` and the +``*.list`` and ``*.sources`` files of ``sources.list.d``, wherever ``apt-config`` +says they are -- with its file, line, types, URI, suites and components. +Commented-out lines and deb822 paragraphs with ``Enabled: no`` are not sources. +Whether a suite names another release than the host's is not said here: which +words are releases of which distribution is the caller's to know +(:func:`suite_release` gives the word). + +**Moving a file** (:meth:`AptReleaseMixin.set_apt_source_release`): every suite of +*release* in one of apt's source files becomes the same suite of *codename* +(``focal-security`` → ``noble-security``), on enabled lines only. Before anything is +written, each new suite's ``InRelease`` (or ``Release``) is fetched from its URI with +apt's own downloader and the proxy apt is set to; when one of them is not there, +the vendor does not build for *codename* and the file stays as it is. The file is +written whole, as root, mode 644. Loading the package index afterwards is the +caller's (``refresh_available_updates``). +""" + +from __future__ import annotations + +import base64 +import re +import shlex +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from typing_extensions import TypedDict + +from napalm_device_types.fixes import run_fix_steps +from napalm_device_types.models import FixResultDict, FixStepOutcomeDict +from napalm_device_types.terminal import strip_terminal_codes + +_BEGIN = "ARELEASE_BEGIN" +_END = "ARELEASE_END" + +#: A release's codename: one lower-case word (``noble``, ``bookworm``). +_CODENAME = re.compile(r"^[a-z][a-z0-9]*$") +#: A suite that names a release: its codename, then perhaps a suffix (``-security``). +_SUITE = re.compile(r"^([a-z][a-z0-9]*)(-[a-z0-9-]+)?$") +#: One source line of a ``.list`` file, its comment cut off: types, options, URI, suite. +_LIST_LINE = re.compile(r"^(\s*(deb|deb-src)\s+(?:\[[^\]]*\]\s+)?(\S+)\s+)(\S+)(.*)$") +#: A deb822 field's first line. +_FIELD = re.compile(r"^([A-Za-z][A-Za-z0-9-]*):(.*)$") +_DISABLED = frozenset({"no", "false", "0"}) + + +class AptSourceDict(TypedDict): + """One source apt reads (``get_apt_source_releases``).""" + + #: The file it is in. + file: str + #: The line of a ``.list`` file's entry; of a deb822 paragraph's ``Suites:`` field. + line: int + #: ``deb``, ``deb-src``; a deb822 paragraph may name both. + types: list[str] + #: Its URI, as written; a deb822 paragraph with several URIs is one source per URI. + uri: str + #: Its suites; a flat repository's path (``./``) is one too. + suites: list[str] + components: list[str] + + +class AptReleasesDict(TypedDict): + """The host's release and its sources (``get_apt_source_releases``).""" + + #: ``ID`` and ``ID_LIKE`` of ``/etc/os-release``; "" and [] when it says none. + os_id: str + os_like: list[str] + #: ``VERSION_CODENAME``; a derivative's own (Linux Mint's ``virginia``). + codename: str + #: ``UBUNTU_CODENAME``: the Ubuntu release Ubuntu and its derivatives are built on. + ubuntu_codename: str + sources: list[AptSourceDict] + + +class AptReleaseFixDict(FixResultDict): + """The result of ``set_apt_source_release``: the fix, and how many suites moved.""" + + rewritten: int + + +def apt_releases_command( + apt_config: str = "apt-config", os_release: str = "/etc/os-release" +) -> str: + """The read-only command that prints the host's release and apt's source files. + + One line, POSIX ``sh``, no privileges. Every line of a file is printed behind a + ``|``, so an empty line -- which ends a deb822 paragraph -- comes through. + *apt_config* and *os_release* let a test point it elsewhere. + """ + return ( + f"( export LC_ALL=C; printf '%s%s\\n' ARELEASE_ BEGIN; echo '@@os'; " + f'o={shlex.quote(os_release)}; [ -r "$o" ] || o=/usr/lib/os-release; ' + '[ -r "$o" ] && sed \'s/^/|/\' "$o"; ' + f'eval "$({apt_config} shell sl Dir::Etc::sourcelist/f sp Dir::Etc::sourceparts/d ' + '2>/dev/null)"; sl="${sl:-/etc/apt/sources.list}"; sp="${sp:-/etc/apt/sources.list.d/}"; ' + 'for f in "$sl" "$sp"*.list "$sp"*.sources; do [ -f "$f" ] || continue; ' + 'if [ -r "$f" ]; then printf \'@@file %s\\n\' "$f"; sed \'s/^/|/\' "$f"; ' + "else printf '@@unreadable %s\\n' \"$f\"; fi; done; " + "printf '%s%s\\n' ARELEASE_ END ) 2>/dev/null | cat" + ) + + +#: The command as a driver runs it. +APT_RELEASES_COMMAND = apt_releases_command() + + +def suite_release(suite: str) -> str | None: + """The word a suite starts with, ``noble`` of ``noble-security``; None for a path. + + Whether that word is a release, and of which distribution, the caller knows: + ``stable`` and ``nodistro`` have the form too. + """ + found = _SUITE.match(suite) + return found.group(1) if found else None + + +def _codename(name: str) -> str: + if not _CODENAME.match(name): + raise ValueError(f"not a release's codename: {name!r}") + return name + + +# ── Reading ────────────────────────────────────────────────────────────────── + + +def _blocks(output: str) -> list[tuple[str, list[str]]]: + """``(header, lines)`` of each block between the frame markers, in order.""" + lines = strip_terminal_codes(output).splitlines() + stripped = [line.strip() for line in lines] + try: + start = stripped.index(_BEGIN) + end = stripped.index(_END, start) + except ValueError: + raise ValueError("no intact apt release report in the output") from None + blocks: list[tuple[str, list[str]]] = [] + for line in lines[start + 1 : end]: + if line.startswith("@@"): + blocks.append((line[2:].strip(), [])) + elif line.startswith("|") and blocks: + blocks[-1][1].append(line[1:].rstrip("\r")) + return blocks + + +def _os_release(lines: list[str]) -> dict[str, str]: + values: dict[str, str] = {} + for line in lines: + key, sep, value = line.strip().partition("=") + if sep and key: + values[key] = value.strip().strip("'\"") + return values + + +def _code(line: str) -> str: + """*line* without its comment.""" + return line.split("#", 1)[0] + + +def _list_sources(path: str, lines: list[str]) -> list[AptSourceDict]: + found: list[AptSourceDict] = [] + for number, line in enumerate(lines, start=1): + entry = _LIST_LINE.match(_code(line)) + if entry: + _, kind, uri, suite, rest = entry.groups() + found.append( + { + "file": path, + "line": number, + "types": [kind], + "uri": uri, + "suites": [suite], + "components": rest.split(), + } + ) + return found + + +@dataclass +class _Field: + first: int # index of its first line + lines: list[int] = field(default_factory=list) # indices of every line it spans + value: str = "" + + +@dataclass +class _Paragraph: + fields: dict[str, _Field] = field(default_factory=dict) + + def value(self, name: str) -> str: + found = self.fields.get(name) + return found.value if found else "" + + @property + def enabled(self) -> bool: + return self.value("enabled").strip().lower() not in _DISABLED + + +def _paragraphs(lines: list[str]) -> list[_Paragraph]: + """The deb822 paragraphs of *lines*; comments are skipped, an empty line ends one.""" + paragraphs: list[_Paragraph] = [] + current: _Paragraph | None = None + last: _Field | None = None + for index, line in enumerate(lines): + if not line.strip(): + current, last = None, None + continue + if line.lstrip().startswith("#"): + continue + if line[0] in " \t": + if last is not None: + last.lines.append(index) + last.value += " " + line.strip() + continue + found = _FIELD.match(line) + if not found: + continue + if current is None: + current = _Paragraph() + paragraphs.append(current) + last = _Field(first=index, lines=[index], value=found.group(2).strip()) + current.fields[found.group(1).lower()] = last + return paragraphs + + +def _deb822_sources(path: str, lines: list[str]) -> list[AptSourceDict]: + found: list[AptSourceDict] = [] + for paragraph in _paragraphs(lines): + suites = paragraph.fields.get("suites") + if not paragraph.enabled or suites is None: + continue + for uri in paragraph.value("uris").split(): + found.append( + { + "file": path, + "line": suites.first + 1, + "types": paragraph.value("types").split(), + "uri": uri, + "suites": suites.value.split(), + "components": paragraph.value("components").split(), + } + ) + return found + + +def _is_deb822(path: str) -> bool: + return path.endswith(".sources") + + +def parse_apt_releases(output: str) -> AptReleasesDict: + """What :func:`apt_releases_command` printed, as :class:`AptReleasesDict`. + + :raises ValueError: if the output carried no intact report. + """ + os_values: dict[str, str] = {} + sources: list[AptSourceDict] = [] + for header, lines in _blocks(output): + kind, _, path = header.partition(" ") + if kind == "os": + os_values = _os_release(lines) + elif kind == "file" and path: + read = _deb822_sources if _is_deb822(path) else _list_sources + sources.extend(read(path, lines)) + return { + "os_id": os_values.get("ID", "").lower(), + "os_like": os_values.get("ID_LIKE", "").lower().split(), + "codename": os_values.get("VERSION_CODENAME", "").lower(), + "ubuntu_codename": os_values.get("UBUNTU_CODENAME", "").lower(), + "sources": sources, + } + + +# ── Rewriting ──────────────────────────────────────────────────────────────── + + +def _moved(suite: str, release: str, codename: str) -> str: + return codename + suite[len(release) :] if suite_release(suite) == release else suite + + +def _rewrite_list_line(line: str, release: str, codename: str) -> tuple[str, int]: + code = _code(line) + entry = _LIST_LINE.match(code) + if not entry: + return line, 0 + suite = entry.group(4) + new = _moved(suite, release, codename) + if new == suite: + return line, 0 + head = entry.group(1) + return head + new + line[len(head) + len(suite) :], 1 + + +def _rewrite_tokens(text: str, release: str, codename: str) -> tuple[str, int]: + changed = 0 + + def one(token: re.Match[str]) -> str: + nonlocal changed + new = _moved(token.group(), release, codename) + changed += new != token.group() + return new + + return re.sub(r"\S+", one, text), changed + + +def _rewrite_deb822(lines: list[str], release: str, codename: str) -> int: + changed = 0 + for paragraph in _paragraphs(lines): + suites = paragraph.fields.get("suites") + if not paragraph.enabled or suites is None: + continue + for index in suites.lines: + line = lines[index] + cut = line.index(":") + 1 if index == suites.first else 0 + tail, count = _rewrite_tokens(line[cut:], release, codename) + lines[index] = line[:cut] + tail + changed += count + return changed + + +def rewrite_release(text: str, release: str, codename: str, *, deb822: bool) -> tuple[str, int]: + """*text*, a source file, with every suite of *release* on *codename*; and how many. + + Only enabled sources: not a commented-out line, not a paragraph with + ``Enabled: no``. A URI or comment that names *release* stays as it is. + + :raises ValueError: if *release* or *codename* is not a codename. + """ + release, codename = _codename(release), _codename(codename) + lines = text.split("\n") + if deb822: + changed = _rewrite_deb822(lines, release, codename) + else: + changed = 0 + for index, line in enumerate(lines): + lines[index], count = _rewrite_list_line(line, release, codename) + changed += count + return "\n".join(lines), changed + + +# ── The mixin ──────────────────────────────────────────────────────────────── + + +def _release_urls(uri: str, suite: str) -> list[str]: + base = f"{uri.rstrip('/')}/dists/{suite}/" + return [base + "InRelease", base + "Release"] + + +class AptReleaseMixin: + """Reads the release each apt source names; moves one file's sources to another. + + The template form, like :class:`~napalm_device_types.apt_proxy.AptProxyMixin`, + which brings it to every driver that has it: a driver supplies + ``_run_apt_command`` and ``_write_apt_file``, and ``_is_apt_host`` where it + can be something else. + """ + + if TYPE_CHECKING: # pragma: no cover - declared for type checkers only + + def _run_apt_command(self, command: str) -> str: + """Run *command* on the host with ``sh``, without privileges; what it printed.""" + ... + + def _write_apt_file(self, path: str, content: str) -> FixStepOutcomeDict: + """Replace *path* with *content*, as a new file owned by root, mode 644.""" + ... + + def _is_apt_host(self) -> bool: + """Whether apt is the host's package manager; a driver that can tell overrides it.""" + return True + + def get_apt_source_releases(self) -> AptReleasesDict | None: + """The host's release and every source apt reads; None on a host without apt. + + :raises ValueError: if the host's output carried no intact report. + """ + if not self._is_apt_host(): + return None + return parse_apt_releases(self._run_apt_command(APT_RELEASES_COMMAND)) + + def set_apt_source_release(self, path: str, release: str, codename: str) -> AptReleaseFixDict: + """Every suite of *release* in *path* on *codename*, where the vendor serves it. + + Three steps: ``read`` (*path* is one of apt's source files, and which of its + sources name *release*), ``check`` (each new suite's ``InRelease`` or + ``Release`` is there) and ``rewrite``. Nothing is written when a step fails; + nothing is fetched or written when no source of *path* names *release*. + """ + moving: list[AptSourceDict] = [] + rewritten = [0] + result = run_fix_steps( + [ + ("read", lambda: self._sources_on(path, release, codename, moving)), + ("check", lambda: self._served(moving, release, codename)), + ("rewrite", lambda: self._rewrite(path, release, codename, moving, rewritten)), + ] + ) + return {**result, "rewritten": rewritten[0]} + + def _sources_on( + self, path: str, release: str, codename: str, moving: list[AptSourceDict] + ) -> FixStepOutcomeDict: + _codename(release) + _codename(codename) + found = self.get_apt_source_releases() + if found is None: + return {"ok": False, "output": "not an apt host"} + in_file = [s for s in found["sources"] if s["file"] == path] + if not in_file: + return {"ok": False, "output": f"{path} is none of apt's source files"} + moving.extend(s for s in in_file if any(suite_release(x) == release for x in s["suites"])) + if not moving: + return {"ok": True, "output": f"no source in {path} is on {release} any more"} + named = "; ".join(f"{s['uri']} {' '.join(s['suites'])}" for s in moving) + return {"ok": True, "output": f"on {release} in {path}: {named}"} + + def _served( + self, moving: list[AptSourceDict], release: str, codename: str + ) -> FixStepOutcomeDict: + from napalm_device_types.apt_proxy import apt_fetch_command, parse_apt_fetch + + wanted = sorted( + { + (s["uri"], _moved(suite, release, codename)) + for s in moving + for suite in s["suites"] + if suite_release(suite) == release + } + ) + said: list[str] = [] + for uri, suite in wanted: + error = "" + for url in _release_urls(uri, suite): + fetched = parse_apt_fetch(self._run_apt_command(apt_fetch_command(url))) + error = fetched["error"] + if not error: + said.append(f"{uri} serves {suite}") + break + else: + return {"ok": False, "output": f"{uri} has no {suite}: {error}; nothing changed"} + return {"ok": True, "output": "; ".join(said) or "nothing to check"} + + def _rewrite( + self, + path: str, + release: str, + codename: str, + moving: list[AptSourceDict], + rewritten: list[int], + ) -> FixStepOutcomeDict: + if not moving: + return {"ok": True, "output": "nothing to rewrite"} + encoded = "".join(self._run_apt_command(f"base64 {shlex.quote(path)}").split()) + text = base64.b64decode(encoded).decode() + new, count = rewrite_release(text, release, codename, deb822=_is_deb822(path)) + if not count: + return {"ok": True, "output": f"no source in {path} is on {release} any more"} + written = self._write_apt_file(path, new) + if not written["ok"]: + return written + rewritten[0] = count + suites = "1 suite" if count == 1 else f"{count} suites" + return {"ok": True, "output": f"{path}: {suites} moved from {release} to {codename}"} + + +__all__ = [ + "APT_RELEASES_COMMAND", + "AptReleaseFixDict", + "AptReleaseMixin", + "AptReleasesDict", + "AptSourceDict", + "apt_releases_command", + "parse_apt_releases", + "rewrite_release", + "suite_release", +] diff --git a/tests/test_apt_releases.py b/tests/test_apt_releases.py new file mode 100644 index 0000000..3a1a8e4 --- /dev/null +++ b/tests/test_apt_releases.py @@ -0,0 +1,393 @@ +"""apt's sources and the release they name, and moving one file's sources to the host's. + +A release upgrade keeps third-party sources on the old release: z2m-house, Ubuntu +24.04 (noble), kept Docker's source on ``focal`` and never saw a Docker newer than +the last focal build (NetOrk/netork#2614). The read command runs here for real, on +files in a temporary directory that ``apt-config -o`` points apt's settings at. +""" + +from __future__ import annotations + +import base64 +import shutil +import subprocess + +import pytest + +from napalm_device_types.apt_proxy import AptProxyMixin +from napalm_device_types.apt_releases import ( + AptReleaseMixin, + apt_releases_command, + parse_apt_releases, + rewrite_release, + suite_release, +) + +_APT_CONFIG = shutil.which("apt-config") +needs_apt = pytest.mark.skipif(_APT_CONFIG is None, reason="apt-config is not on this host") + +_NOBLE = ( + 'PRETTY_NAME="Ubuntu 24.04.3 LTS"\nNAME="Ubuntu"\nVERSION_ID="24.04"\n' + "VERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nUBUNTU_CODENAME=noble\n" +) +_TRIXIE = 'PRETTY_NAME="Debian GNU/Linux 13 (trixie)"\nVERSION_CODENAME=trixie\nID=debian\n' +_MINT = ( + 'NAME="Linux Mint"\nVERSION_CODENAME=virginia\nID=linuxmint\nID_LIKE="ubuntu debian"\n' + "UBUNTU_CODENAME=jammy\n" +) + +#: z2m-house's Docker source, as found: through the cache, on focal (#2614). +_Z2M_DOCKER = "deb http://HTTPS///download.docker.com/linux/ubuntu focal stable\n" +_Z2M_DOCKER_FILE = "archive_uri-https_download_docker_com_linux_ubuntu-noble.list" +_UBUNTU_SOURCES = ( + "Types: deb\n" + "URIs: http://archive.ubuntu.com/ubuntu/\n" + "Suites: noble noble-updates noble-backports\n" + "Components: main restricted universe multiverse\n" + "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n" + "\n" + "Types: deb\n" + "URIs: http://security.ubuntu.com/ubuntu/\n" + "Suites: noble-security\n" + "Components: main restricted universe multiverse\n" +) + + +def _framed(os_release: str, files: dict[str, str]) -> str: + """What :func:`apt_releases_command` prints for *os_release* and *files*.""" + lines = ["ARELEASE_BEGIN", "@@os"] + lines += [f"|{line}" for line in os_release.splitlines()] + for path, text in files.items(): + lines.append(f"@@file {path}") + lines += [f"|{line}" for line in text.splitlines()] + lines.append("ARELEASE_END") + return "\n".join(lines) + "\n" + + +class TestSuiteRelease: + @pytest.mark.parametrize( + ("suite", "release"), + [ + ("noble", "noble"), + ("noble-security", "noble"), + ("noble-updates", "noble"), + ("focal-backports", "focal"), + ("bookworm-proposed-updates", "bookworm"), + ("trixie-security", "trixie"), + ], + ) + def test_the_release_a_suite_names(self, suite, release): + assert suite_release(suite) == release + + @pytest.mark.parametrize( + ("suite", "word"), + [("stable", "stable"), ("nodistro", "nodistro"), ("pve-no-subscription", "pve")], + ) + def test_a_word_the_caller_judges(self, suite, word): + # Which words are releases is the caller's to know: "stable" has the form too. + assert suite_release(suite) == word + + @pytest.mark.parametrize("suite", ["/", "./", "focal/", "Focal", "$(ARCH)", ""]) + def test_a_suite_of_another_form(self, suite): + # A flat repository's path, or anything not a lower-case word, names none. + assert suite_release(suite) is None + + +class TestParse: + def test_the_host_and_a_list_source(self): + found = parse_apt_releases( + _framed(_NOBLE, {f"/etc/apt/sources.list.d/{_Z2M_DOCKER_FILE}": _Z2M_DOCKER}) + ) + assert found["os_id"] == "ubuntu" + assert found["os_like"] == ["debian"] + assert found["codename"] == "noble" + assert found["ubuntu_codename"] == "noble" + assert found["sources"] == [ + { + "file": f"/etc/apt/sources.list.d/{_Z2M_DOCKER_FILE}", + "line": 1, + "types": ["deb"], + "uri": "http://HTTPS///download.docker.com/linux/ubuntu", + "suites": ["focal"], + "components": ["stable"], + } + ] + + def test_options_comments_and_disabled_lines(self): + text = ( + "# deb http://old.example/ubuntu bionic main\n" + "\n" + "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] " + "https://download.docker.com/linux/ubuntu jammy stable # moved by hand\n" + "deb-src [ arch=amd64 ] http://archive.ubuntu.com/ubuntu noble main\n" + "deb https://repo.example/flat ./\n" + ) + found = parse_apt_releases(_framed(_NOBLE, {"/etc/apt/sources.list": text})) + assert [ + (s["line"], s["types"], s["uri"], s["suites"], s["components"]) + for s in found["sources"] + ] == [ + (3, ["deb"], "https://download.docker.com/linux/ubuntu", ["jammy"], ["stable"]), + (4, ["deb-src"], "http://archive.ubuntu.com/ubuntu", ["noble"], ["main"]), + (5, ["deb"], "https://repo.example/flat", ["./"], []), + ] + + def test_deb822_paragraphs_one_per_uri(self): + text = ( + "Types: deb deb-src\n" + "URIs: http://a.example/ubuntu http://b.example/ubuntu\n" + "Suites: focal\n" + " focal-updates\n" + "Components: main\n" + "\n" + "# a comment between paragraphs\n" + "Enabled: no\n" + "Types: deb\n" + "URIs: http://c.example/ubuntu\n" + "Suites: bionic\n" + "Components: main\n" + ) + found = parse_apt_releases(_framed(_NOBLE, {"/etc/apt/sources.list.d/x.sources": text})) + assert [(s["line"], s["types"], s["uri"], s["suites"]) for s in found["sources"]] == [ + (3, ["deb", "deb-src"], "http://a.example/ubuntu", ["focal", "focal-updates"]), + (3, ["deb", "deb-src"], "http://b.example/ubuntu", ["focal", "focal-updates"]), + ] + + def test_debian_and_a_derivative(self): + assert parse_apt_releases(_framed(_TRIXIE, {}))["codename"] == "trixie" + mint = parse_apt_releases(_framed(_MINT, {})) + assert (mint["os_id"], mint["os_like"], mint["codename"], mint["ubuntu_codename"]) == ( + "linuxmint", + ["ubuntu", "debian"], + "virginia", + "jammy", + ) + + def test_without_os_release(self): + found = parse_apt_releases("ARELEASE_BEGIN\n@@os\nARELEASE_END\n") + assert (found["os_id"], found["codename"], found["ubuntu_codename"]) == ("", "", "") + + def test_an_unreadable_file_is_left_out(self): + out = _framed(_NOBLE, {}).replace( + "ARELEASE_END", "@@unreadable /etc/apt/sources.list\nARELEASE_END" + ) + assert parse_apt_releases(out)["sources"] == [] + + def test_no_intact_report(self): + with pytest.raises(ValueError): + parse_apt_releases("ARELEASE_BEGIN\n@@os\n") + + +@needs_apt +class TestTheCommand: + def test_reads_list_and_sources_files_as_apt_names_them(self, tmp_path): + (tmp_path / "sources.list.d").mkdir() + (tmp_path / "sources.list").write_text("") + (tmp_path / "sources.list.d" / _Z2M_DOCKER_FILE).write_text(_Z2M_DOCKER) + (tmp_path / "sources.list.d" / "ubuntu.sources").write_text(_UBUNTU_SOURCES) + (tmp_path / "sources.list.d" / "old.list.distUpgrade").write_text(_Z2M_DOCKER) + (tmp_path / "os-release").write_text(_NOBLE) + command = apt_releases_command( + apt_config=( + f"apt-config -o Dir::Etc::sourcelist={tmp_path}/sources.list " + f"-o Dir::Etc::sourceparts={tmp_path}/sources.list.d/" + ), + os_release=str(tmp_path / "os-release"), + ) + done = subprocess.run(["sh", "-c", command], capture_output=True, text=True, check=False) + found = parse_apt_releases(done.stdout) + assert found["codename"] == "noble" + assert [(s["file"].rsplit("/", 1)[-1], s["suites"]) for s in found["sources"]] == [ + (_Z2M_DOCKER_FILE, ["focal"]), + ("ubuntu.sources", ["noble", "noble-updates", "noble-backports"]), + ("ubuntu.sources", ["noble-security"]), + ] + + +class TestRewriteRelease: + def test_a_list_line(self): + assert rewrite_release(_Z2M_DOCKER, "focal", "noble", deb822=False) == ( + "deb http://HTTPS///download.docker.com/linux/ubuntu noble stable\n", + 1, + ) + + def test_suffixes_options_and_a_uri_that_names_the_release(self): + text = ( + "deb [signed-by=/k.asc] http://repo.example/focal/ focal-security main # focal\n" + "# deb http://repo.example/focal/ focal main\n" + "deb http://repo.example/other noble main\n" + "deb http://repo.example/flat ./\n" + ) + assert rewrite_release(text, "focal", "noble", deb822=False) == ( + "deb [signed-by=/k.asc] http://repo.example/focal/ noble-security main # focal\n" + "# deb http://repo.example/focal/ focal main\n" + "deb http://repo.example/other noble main\n" + "deb http://repo.example/flat ./\n", + 1, + ) + + def test_a_release_that_is_only_a_prefix_is_not_it(self): + text = "deb http://repo.example/x focalfossa main\n" + assert rewrite_release(text, "focal", "noble", deb822=False) == (text, 0) + + def test_deb822_suites_and_continuations_of_enabled_paragraphs(self): + text = ( + "Types: deb\n" + "URIs: http://a.example/focal\n" + "Suites: focal focal-updates\n" + " focal-backports\n" + "Components: main\n" + "\n" + "Enabled: no\n" + "Types: deb\n" + "URIs: http://b.example/ubuntu\n" + "Suites: focal\n" + "Components: main\n" + ) + new, changed = rewrite_release(text, "focal", "noble", deb822=True) + assert changed == 3 + assert new == ( + "Types: deb\n" + "URIs: http://a.example/focal\n" + "Suites: noble noble-updates\n" + " noble-backports\n" + "Components: main\n" + "\n" + "Enabled: no\n" + "Types: deb\n" + "URIs: http://b.example/ubuntu\n" + "Suites: focal\n" + "Components: main\n" + ) + + def test_the_text_without_a_final_newline_stays_so(self): + assert rewrite_release("deb http://a/ focal main", "focal", "noble", deb822=False) == ( + "deb http://a/ noble main", + 1, + ) + + @pytest.mark.parametrize("bad", ["", "Noble", "no ble", "noble;", "../x"]) + def test_a_name_that_is_no_codename(self, bad): + with pytest.raises(ValueError): + rewrite_release(_Z2M_DOCKER, bad, "noble", deb822=False) + with pytest.raises(ValueError): + rewrite_release(_Z2M_DOCKER, "focal", bad, deb822=False) + + +_OK = "[fetch]\nGet:1 x\n[rc]\n0\n" +_NOT_FOUND = "[fetch]\nE: Failed to fetch {url} 404 Not Found [IP: 1.2.3.4 80]\n[rc]\n100\n" + + +class _Host(AptReleaseMixin): + """An apt host whose files are a dict and whose fetches answer from a set.""" + + def __init__(self, files, os_release=_NOBLE, served=(), apt=True): + self.files = dict(files) + self.os_release = os_release + self.served = set(served) + self.apt = apt + self.fetched: list[str] = [] + self.written: list[str] = [] + + def _is_apt_host(self): + return self.apt + + def _run_apt_command(self, command): + if "ARELEASE_" in command: + return _framed(self.os_release, self.files) + if command.startswith("base64 "): + path = command.split(" ", 1)[1].strip("'") + return base64.b64encode(self.files[path].encode()).decode() + "\n" + url = command.split("download-file ", 1)[1].split()[0].strip("'") + self.fetched.append(url) + body = _OK if url in self.served else _NOT_FOUND.format(url=url) + return f"APROXY_BEGIN\n{body}APROXY_END\n" + + def _write_apt_file(self, path, content): + self.written.append(path) + self.files[path] = content + return {"ok": True, "output": f"wrote {path}"} + + +_DOCKER_PATH = f"/etc/apt/sources.list.d/{_Z2M_DOCKER_FILE}" +_DOCKER_NOBLE = "http://HTTPS///download.docker.com/linux/ubuntu/dists/noble/InRelease" + + +class TestTheMixin: + def test_reads_the_host(self): + found = _Host({_DOCKER_PATH: _Z2M_DOCKER}).get_apt_source_releases() + assert found is not None + assert found["codename"] == "noble" + assert found["sources"][0]["suites"] == ["focal"] + + def test_none_without_apt(self): + assert _Host({}, apt=False).get_apt_source_releases() is None + + def test_moves_the_file_once_the_vendor_serves_the_release(self): + host = _Host({_DOCKER_PATH: _Z2M_DOCKER}, served={_DOCKER_NOBLE}) + result = host.set_apt_source_release(_DOCKER_PATH, "focal", "noble") + assert result["success"], result["output"] + assert host.files[_DOCKER_PATH] == ( + "deb http://HTTPS///download.docker.com/linux/ubuntu noble stable\n" + ) + assert host.fetched == [_DOCKER_NOBLE] + assert result["rewritten"] == 1 + assert [s["step"] for s in result["steps"]] == ["read", "check", "rewrite"] + + def test_the_release_file_when_there_is_no_inrelease(self): + release = _DOCKER_NOBLE.replace("InRelease", "Release") + host = _Host({_DOCKER_PATH: _Z2M_DOCKER}, served={release}) + assert host.set_apt_source_release(_DOCKER_PATH, "focal", "noble")["success"] + assert host.fetched == [_DOCKER_NOBLE, release] + + def test_refuses_where_the_vendor_has_no_such_release(self): + host = _Host({_DOCKER_PATH: _Z2M_DOCKER}) + result = host.set_apt_source_release(_DOCKER_PATH, "focal", "noble") + assert not result["success"] + assert host.written == [] + assert "noble" in result["output"] and "404" in result["output"] + assert result["rewritten"] == 0 + + def test_one_suite_missing_leaves_the_whole_file(self): + text = "deb http://a.example/u focal main\ndeb http://a.example/u focal-security main\n" + served = {"http://a.example/u/dists/noble/InRelease"} + host = _Host({"/etc/apt/sources.list": text}, served=served) + result = host.set_apt_source_release("/etc/apt/sources.list", "focal", "noble") + assert not result["success"] + assert host.files["/etc/apt/sources.list"] == text + + def test_a_file_apt_does_not_read_is_never_written(self): + host = _Host({_DOCKER_PATH: _Z2M_DOCKER}, served={_DOCKER_NOBLE}) + result = host.set_apt_source_release("/etc/shadow", "focal", "noble") + assert not result["success"] + assert host.written == [] + + def test_nothing_left_on_that_release(self): + host = _Host({_DOCKER_PATH: _Z2M_DOCKER.replace("focal", "noble")}) + result = host.set_apt_source_release(_DOCKER_PATH, "focal", "noble") + assert result["success"] + assert result["rewritten"] == 0 + assert host.written == [] and host.fetched == [] + + def test_a_deb822_file(self): + path = "/etc/apt/sources.list.d/docker.sources" + text = "Types: deb\nURIs: https://download.docker.com/linux/ubuntu\nSuites: focal\nComponents: stable\n" + served = {"https://download.docker.com/linux/ubuntu/dists/noble/InRelease"} + host = _Host({path: text}, served=served) + assert host.set_apt_source_release(path, "focal", "noble")["success"] + assert "Suites: noble\n" in host.files[path] + + def test_not_an_apt_host(self): + result = _Host({}, apt=False).set_apt_source_release(_DOCKER_PATH, "focal", "noble") + assert not result["success"] + + def test_a_name_that_is_no_codename_changes_nothing(self): + host = _Host({_DOCKER_PATH: _Z2M_DOCKER}, served={_DOCKER_NOBLE}) + result = host.set_apt_source_release(_DOCKER_PATH, "focal", "noble; rm -rf /") + assert not result["success"] + assert host.written == [] and host.fetched == [] + + +def test_every_apt_proxy_host_reads_its_releases(): + # Every driver with apt's proxy is an apt host: it reads its sources' releases too. + assert issubclass(AptProxyMixin, AptReleaseMixin)