diff --git a/README.md b/README.md index be4dc0f..823bb58 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ with its CIS result and SBOM attached (NetOrk/netork#1390). | **Signature** (`bin/sign.sh`) | A cosign signature on the reference and on every platform image behind it, with the CIS result and the SBOM as in-toto attestations. Nothing goes to a public transparency log. | | **Verification** (`bin/verify-base.sh`, `bin/verify-signature.sh`) | Every `FROM` image of a Dockerfile is pinned by digest and signed with the key in `cosign.pub`. | | **Verdict** (`image_tools/verify.py`) | Whether a rebuild improves on the pinned image, by netOrk's matcher (kb-server), for a base's nightly. | +| **Bump** (`image_tools/bump.py`) | One pull request in every repository of a `dependents.toml` that moves only the pin, and that repository's workflow started. | ## Using it in a workflow diff --git a/actions/bump/action.yml b/actions/bump/action.yml new file mode 100644 index 0000000..bacfa2f --- /dev/null +++ b/actions/bump/action.yml @@ -0,0 +1,35 @@ +name: Move the pin in every dependent +description: >- + Open or update one pull request in every repository of dependents.toml that + moves only the pin, and start that repository's workflow (image_tools/bump.py). + The caller passes BUMP_TOKEN, the netork-bot account's token, as env. +inputs: + image: + description: The image whose pin moves, e.g. netork/base-os + required: true + ref: + description: The new reference, :@ + required: true + version: + description: The new version + required: true + report: + description: The pull requests' body, a file + default: report.md + dependents: + description: The repositories that pin the image + default: dependents.toml +runs: + using: composite + steps: + - shell: bash + env: + IMAGE: ${{ inputs.image }} + REF: ${{ inputs.ref }} + VERSION: ${{ inputs.version }} + REPORT: ${{ inputs.report }} + DEPENDENTS: ${{ inputs.dependents }} + PYTHONPATH: ${{ github.action_path }}/../.. + run: | + python3 -m image_tools.bump "$IMAGE" "$REF" "$VERSION" \ + --report "$REPORT" --dependents "$DEPENDENTS" diff --git a/actions/verify-base/action.yml b/actions/verify-base/action.yml index c154450..01f83de 100644 --- a/actions/verify-base/action.yml +++ b/actions/verify-base/action.yml @@ -1,11 +1,17 @@ name: Verify the base images description: >- - Every image the Dockerfiles build on is signed with netOrk's key, checked - before the build (NetOrk/netork#1390, bin/verify-base.sh). + Every image the Dockerfiles build on, and every image named in refs, is + signed with netOrk's key, checked before the build (NetOrk/netork#1390, + bin/verify-base.sh, bin/verify-signature.sh). Give one or both. inputs: dockerfiles: - description: The Dockerfiles, separated by spaces, relative to the workspace - required: true + description: Dockerfiles whose FROM images to verify, separated by spaces + default: "" + refs: + description: >- + Images to verify, @, separated by spaces: for a FROM the + Dockerfile only knows as an argument + default: "" runs: using: composite steps: @@ -16,5 +22,18 @@ runs: shell: bash env: DOCKERFILES: ${{ inputs.dockerfiles }} - # shellcheck disable=SC2086 # the list splits on purpose - run: bash "${{ github.action_path }}/../../bin/verify-base.sh" $DOCKERFILES + REFS: ${{ inputs.refs }} + # The lists split on spaces on purpose. + run: | + if [ -z "$DOCKERFILES$REFS" ]; then + echo "verify-base: give dockerfiles, refs or both" >&2 + exit 1 + fi + if [ -n "$DOCKERFILES" ]; then + # shellcheck disable=SC2086 + bash "${{ github.action_path }}/../../bin/verify-base.sh" $DOCKERFILES + fi + if [ -n "$REFS" ]; then + # shellcheck disable=SC2086 + bash "${{ github.action_path }}/../../bin/verify-signature.sh" $REFS + fi diff --git a/image_tools/bump.py b/image_tools/bump.py new file mode 100644 index 0000000..8db0e90 --- /dev/null +++ b/image_tools/bump.py @@ -0,0 +1,209 @@ +"""Move a pin in the repos that use an image, and start their own run (#1308). + +When an image improves, its repo opens or updates one pull request in every +dependent (``dependents.toml``), rewriting only the pin, then dispatches the +dependent's workflow, which rebuilds on the new pin and verifies itself. + + python -m image_tools.bump --report report.md + +Talks to Gitea with ``BUMP_TOKEN`` (the netork-bot account); standard library only. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import tomllib +import urllib.error +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +GITEA_URL = os.environ.get("GITEA_URL", "https://git.netork.io") +REGISTRY = "registry.netork.io" +#: A pinned reference: registry/repo:tag@sha256:<64 hex>. +_REF = re.compile(r"^registry\.netork\.io/[a-z0-9./-]+:[A-Za-z0-9._-]+@sha256:[0-9a-f]{64}$") + +Api = Callable[..., object] + + +@dataclass(frozen=True) +class Dependent: + repo: str + files: tuple[str, ...] + workflow: str + + +def load_dependents(path: Path) -> list[Dependent]: + data = tomllib.loads(path.read_text()) + return [ + Dependent(d["repo"], tuple(d["files"]), d["workflow"]) for d in data.get("dependent", []) + ] + + +def rewrite_pin(text: str, image: str, new_ref: str) -> str: + """*text* with every pinned reference of *image* replaced by *new_ref*.""" + if not _REF.match(new_ref): + raise ValueError(f"not a pinned reference: {new_ref!r}") + # The host may be written as ${REGISTRY_HOST:-registry.netork.io} (netork's + # compose files); that spelling stays, only the path, tag and digest move. + pattern = re.compile( + rf"({re.escape(REGISTRY)}\}}?/){re.escape(image)}:[A-Za-z0-9._-]+@sha256:[0-9a-f]{{64}}" + ) + tail = new_ref.split("/", 1)[1] + return pattern.sub(lambda m: m.group(1) + tail, text) + + +def _branch(image: str, version: str) -> str: + return f"chore/{image.rsplit('/', 1)[-1]}-{version}" + + +def open_or_update_pr( + api: Api, dep: Dependent, image: str, new_ref: str, version: str, report: str +) -> dict | None: + """One PR in *dep* moving *image* to *new_ref*; None when no file pins it. + + A second run for the same version works on the branch the first one made: + files already moved stay as they are, and the open PR gets the new report. + """ + owner_repo = f"/repos/{dep.repo}" + # Gitea answers 404 for a repo the token's account may not see, and for its + # files too; that read as "no pin to move" for every dependent (2026-10-22). + try: + api("GET", owner_repo) + except FileNotFoundError: + raise PermissionError(f"{dep.repo} is not visible to the bump token's account") from None + branch = _branch(image, version) + try: + api("GET", f"{owner_repo}/branches/{branch}") + branch_exists = True + except (FileNotFoundError, urllib.error.HTTPError): + branch_exists = False + ref = branch if branch_exists else "main" + + changes = {} + for name in dep.files: + try: + current = api("GET", f"{owner_repo}/contents/{name}?ref={ref}") + except FileNotFoundError: + continue # not there yet (a repo still being set up): no pin to move + text = base64.b64decode(current["content"]).decode() + moved = rewrite_pin(text, image, new_ref) + if moved != text: + changes[name] = (moved, current["sha"]) + if not changes and not branch_exists: + return None + + if not branch_exists: + api( + "POST", f"{owner_repo}/branches", {"new_branch_name": branch, "old_branch_name": "main"} + ) + title = f"chore(images): move {image} to {version}" + for name, (text, sha) in changes.items(): + api( + "PUT", + f"{owner_repo}/contents/{name}", + { + "content": base64.b64encode(text.encode()).decode(), + "sha": sha, + "branch": branch, + "message": title, + }, + ) + + open_prs = api("GET", f"{owner_repo}/pulls?state=open&limit=50") + for pr in open_prs: + if pr["head"]["ref"] == branch: + return api("PATCH", f"{owner_repo}/pulls/{pr['number']}", {"body": report}) + return api( + "POST", + f"{owner_repo}/pulls", + {"head": branch, "base": "main", "title": title, "body": report}, + ) + + +def dispatch(api: Api, dep: Dependent, inputs: dict[str, str]) -> None: + """Start *dep*'s own workflow on main; one without a workflow gets the PR only.""" + if not dep.workflow: + return + api( + "POST", + f"/repos/{dep.repo}/actions/workflows/{dep.workflow}/dispatches", + {"ref": "main", "inputs": inputs}, + ) + + +def bump_all( + api: Api, + deps: list[Dependent], + image: str, + new_ref: str, + version: str, + report: str, + say: Callable[[str], None], +) -> bool: + """A pull request in every dependent and its run started; False when one failed. + + A repo the token cannot see, or a workflow that does not exist, is named and + the others go on: one wrong entry must not keep the rest at the old pin. + """ + ok = True + for dep in deps: + try: + pr = open_or_update_pr(api, dep, image, new_ref, version, report) + except PermissionError as exc: + say(f"{dep.repo}: {exc}") + ok = False + continue + say(f"{dep.repo}: {'PR #' + str(pr['number']) if pr else 'no pin to move'}") + if not pr: + continue + try: + dispatch(api, dep, {"base_ref": new_ref}) + except FileNotFoundError: + say(f"{dep.repo}: workflow {dep.workflow} not found; its pull request stands") + ok = False + return ok + + +def gitea(method: str, path: str, body: dict | None = None): # pragma: no cover - network seam + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request( + f"{GITEA_URL}/api/v1{path}", + data=data, + method=method, + headers={ + "Authorization": f"token {os.environ['BUMP_TOKEN']}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 - https + raw = response.read() + except urllib.error.HTTPError as exc: + if exc.code == 404: + raise FileNotFoundError(path) from exc + raise + return json.loads(raw) if raw else {} + + +def main() -> int: # pragma: no cover - CLI seam + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("image", help="e.g. netork/base-os") + parser.add_argument("new_ref") + parser.add_argument("version") + parser.add_argument("--report", required=True) + parser.add_argument("--dependents", default="dependents.toml") + args = parser.parse_args() + report = Path(args.report).read_text() + deps = load_dependents(Path(args.dependents)) + ok = bump_all(gitea, deps, args.image, args.new_ref, args.version, report, print) + return 0 if ok else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_actions.py b/tests/test_actions.py index 3cad5c9..7d76753 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -17,8 +17,10 @@ ROOT = Path(__file__).resolve().parents[1] ACTIONS = sorted((ROOT / "actions").glob("*/action.yml")) -def test_the_four_actions(): - assert [a.parent.name for a in ACTIONS] == ["judge", "sign", "verdict", "verify-base"] +def test_the_actions(): + names = [a.parent.name for a in ACTIONS] + + assert names == ["bump", "judge", "sign", "verdict", "verify-base"] @pytest.mark.parametrize("action", ACTIONS, ids=lambda a: a.parent.name) @@ -46,3 +48,21 @@ def test_a_verdict_that_could_not_be_reached_fails_the_step(): assert spec["outputs"]["improved"]["value"] == "${{ steps.verdict.outputs.improved }}" assert "['error']" in step["run"] and "sys.exit(" in step["run"] assert "cat report.md" in step["run"].splitlines() + + +def test_bumping_needs_its_token_from_the_caller(): + text = (ROOT / "actions/bump/action.yml").read_text() + + assert "BUMP_TOKEN" in text and "secrets." not in text + + +def test_a_base_can_be_named_where_the_dockerfile_cannot(): + """base-python builds FROM ${BASE_OS}, an argument the nightly sets, and copies + CPython from Docker's python:3.12-slim, which is not signed with netOrk's key. + It names the one image to verify instead.""" + spec = yaml.safe_load((ROOT / "actions/verify-base/action.yml").read_text()) + runs = "\n".join(step["run"] for step in spec["runs"]["steps"]) + + assert not spec["inputs"]["dockerfiles"].get("required") + assert "refs" in spec["inputs"] + assert "bin/verify-signature.sh" in runs and "bin/verify-base.sh" in runs diff --git a/tests/test_bump.py b/tests/test_bump.py new file mode 100644 index 0000000..2da1cc7 --- /dev/null +++ b/tests/test_bump.py @@ -0,0 +1,167 @@ +"""Moving a pin in the repos that use an image (#1308). + +When an image improves, its repo opens (or updates) one pull request in every +repo that pins it, rewriting only the pin, and starts that repo's own nightly +run, which rebuilds on the new pin and verifies itself. +""" + +from __future__ import annotations + +import base64 + + +from image_tools.bump import Dependent, bump_all, load_dependents, open_or_update_pr, rewrite_pin + +OLD = "registry.netork.io/netork/base-os:13-20261001.1@sha256:" + "a" * 64 +NEW = "registry.netork.io/netork/base-os:13-20261020.1@sha256:" + "b" * 64 + + +class TestRewritePin: + def test_every_from_line_moves(self): + text = f"FROM {OLD} AS build\nRUN x\nFROM {OLD}\n" + + assert ( + rewrite_pin(text, "netork/base-os", NEW) == f"FROM {NEW} AS build\nRUN x\nFROM {NEW}\n" + ) + + def test_a_compose_image_line_moves(self): + text = f"services:\n cache:\n image: {OLD}\n" + + assert ( + rewrite_pin(text, "netork/base-os", NEW) == f"services:\n cache:\n image: {NEW}\n" + ) + + def test_another_image_stays(self): + other = "registry.netork.io/netork/base-python:3.12.15-13-20261001.1@sha256:" + "c" * 64 + text = f"FROM {other}\n" + + assert rewrite_pin(text, "netork/base-os", NEW) == text + + def test_an_image_whose_name_only_starts_alike_stays(self): + text = "FROM registry.netork.io/netork/base-os-extra:1@sha256:" + "d" * 64 + "\n" + + assert rewrite_pin(text, "netork/base-os", NEW) == text + + +class FakeGitea: + def __init__(self, files: dict[str, str], prs: list[dict] | None = None, branches=()): + self.files = dict(files) + self.prs = list(prs or []) + self.branches = set(branches) + self.calls: list[tuple[str, str]] = [] + self.missing_workflows: set[str] = set() + self.dispatched: list[str] = [] + + def __call__(self, method: str, path: str, body: dict | None = None): + self.calls.append((method, path)) + if method == "GET" and path.count("/") == 3: # /repos// + return {"full_name": path.split("/repos/", 1)[1]} + if method == "GET" and "/contents/" in path: + name = path.split("/contents/", 1)[1].split("?", 1)[0] + content = base64.b64encode(self.files[name].encode()).decode() + return {"content": content, "sha": f"sha-{name}"} + if method == "PUT" and "/contents/" in path: + name = path.split("/contents/", 1)[1] + self.files[name] = base64.b64decode(body["content"]).decode() + return {} + if method == "POST" and path.endswith("/branches"): + self.branches.add(body["new_branch_name"]) + return {} + if method == "GET" and "/branches/" in path: + name = path.rsplit("/branches/", 1)[1] + if name not in self.branches: + raise FileNotFoundError(name) + return {} + if method == "GET" and "/pulls?" in path: + return [p for p in self.prs if p["state"] == "open"] + if method == "POST" and path.endswith("/pulls"): + pr = {"number": 7, "state": "open", "head": {"ref": body["head"]}, "body": body["body"]} + self.prs.append(pr) + return pr + if method == "POST" and path.endswith("/dispatches"): + workflow = path.split("/workflows/", 1)[1].split("/", 1)[0] + if workflow in self.missing_workflows: + raise FileNotFoundError(path) + self.dispatched.append(workflow) + return {} + if method == "PATCH" and "/pulls/" in path: + self.prs[0]["body"] = body["body"] + return self.prs[0] + raise AssertionError(f"unexpected {method} {path}") + + +DEP = Dependent(repo="NetOrk/base-python", files=("Dockerfile",), workflow="nightly.yml") + + +class TestPullRequest: + def test_it_opens_one_pr_with_only_the_pin_changed(self): + api = FakeGitea({"Dockerfile": f"FROM {OLD}\nRUN echo hi\n"}) + + pr = open_or_update_pr(api, DEP, "netork/base-os", NEW, "13-20261020.1", "report") + + assert pr["number"] == 7 + assert api.files["Dockerfile"] == f"FROM {NEW}\nRUN echo hi\n" + assert "chore/base-os-13-20261020.1" in api.branches + assert api.prs[0]["body"] == "report" + + def test_a_second_run_updates_the_open_pr(self): + branch = "chore/base-os-13-20261020.1" + api = FakeGitea( + {"Dockerfile": f"FROM {NEW}\n"}, + prs=[{"number": 7, "state": "open", "head": {"ref": branch}, "body": "old"}], + branches={branch}, + ) + + open_or_update_pr(api, DEP, "netork/base-os", NEW, "13-20261020.1", "new report") + + assert len(api.prs) == 1 + assert api.prs[0]["body"] == "new report" + assert not [c for c in api.calls if c[0] == "POST" and c[1].endswith("/branches")] + + def test_a_file_without_the_pin_is_left_alone(self): + api = FakeGitea({"Dockerfile": "FROM something-else\n"}) + + assert open_or_update_pr(api, DEP, "netork/base-os", NEW, "13-20261020.1", "r") is None + assert not [c for c in api.calls if c[0] in ("PUT", "POST")] + + +def test_the_dependents_are_listed_in_the_repo(tmp_path): + path = tmp_path / "dependents.toml" + path.write_text( + '[[dependent]]\nrepo = "NetOrk/netork"\nfiles = ["docker/ui/Dockerfile"]\n' + 'workflow = "ui-nightly.yml"\n' + ) + + assert load_dependents(path) == [ + Dependent("NetOrk/netork", ("docker/ui/Dockerfile",), "ui-nightly.yml") + ] + + +class TestEveryDependent: + """2026-10-22: netork's dependents entry named ui-nightly.yml, which does not + exist yet; the dispatch's 404 ended the run with a traceback after the last + pull request. A dependent that cannot be started is named, the others go + on, and the run fails at the end, as for a repo the token cannot see.""" + + DEPS = [ + Dependent(repo="NetOrk/netork", files=("Dockerfile",), workflow="ui-nightly.yml"), + Dependent(repo="NetOrk/valkey", files=("Dockerfile",), workflow="nightly.yml"), + ] + + def test_a_missing_workflow_is_named_and_the_rest_goes_on(self): + api = FakeGitea({"Dockerfile": f"FROM {OLD}\n"}) + api.missing_workflows = {"ui-nightly.yml"} + lines: list[str] = [] + + ok = bump_all(api, self.DEPS, "netork/base-os", NEW, "13-20261020.1", "r", lines.append) + + assert not ok + assert api.dispatched == ["nightly.yml"] + assert any("NetOrk/netork" in ln and "ui-nightly.yml" in ln for ln in lines) + assert any(ln.startswith("NetOrk/valkey: PR #7") for ln in lines) + + def test_all_well_is_ok(self): + api = FakeGitea({"Dockerfile": f"FROM {OLD}\n"}) + + assert bump_all(api, self.DEPS[1:], "netork/base-os", NEW, "13-20261020.1", "r", print) + assert api.dispatched == ["nightly.yml"]