From 476afe3982bba7326fa83832d35410f615e73ead Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Fri, 23 Oct 2026 12:06:08 +0200 Subject: [PATCH] fix: remove only the images the deploy replaced, never every unused one The deploy ended with docker image prune -af: every image no container used at that moment. On a host that runs other things too, that takes their rollback images; the KB's copy of this line did exactly that on compute-01 (NetOrk/kb-server#157). deploy.sh now records the image IDs netOrk's own containers use before the swap and, after the verification and the migration, hands them to remote/remove_replaced_images.sh. It removes each by its tags, or by its ID once a moving tag left it without one, and keeps any image a container still uses: a redeploy of the same tag replaces nothing. --- README.md | 7 +- deploy.sh | 21 ++++- remote/remove_replaced_images.sh | 29 +++++++ tests/test_deploy_replaced_images.py | 80 ++++++++++++++++++ tests/test_remote_replaced_images.py | 120 +++++++++++++++++++++++++++ 5 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 remote/remove_replaced_images.sh create mode 100644 tests/test_deploy_replaced_images.py create mode 100644 tests/test_remote_replaced_images.py diff --git a/README.md b/README.md index f53773e..c795a2d 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,12 @@ started earlier pulls whatever image the registry held before, which is stale. 10. Verifies that the containers run exactly the image that was pulled. If they don't, the deploy fails. 11. Runs `alembic upgrade head` inside `netork-api`. -12. Prunes unused images. +12. Removes the images the swap replaced, and only those. Before the swap it records which + images netOrk's own containers use; afterwards `remote/remove_replaced_images.sh` + removes them unless a container still uses one, a redeploy of the same tag for + example. Other projects' images on the host stay. This used to be + `docker image prune -af`, which took every unused image, theirs included + (NetOrk/kb-server#157). A failing step stops that server's deploy with a non-zero exit, and the other servers carry on. The script exits non-zero if any server failed and names those servers. diff --git a/deploy.sh b/deploy.sh index a396af4..79100fb 100644 --- a/deploy.sh +++ b/deploy.sh @@ -429,6 +429,14 @@ REMOTE_ENV # netOrk was down until someone ran the deploy again, which then went through # (NetOrk/netork#586). The second attempt is that rerun. Should it fail as well, # the container states go to the log and the deploy fails as before. + # The images netOrk's own containers use now, before the swap: after it, these + # are what the deploy replaced, and the only images it removes (below). + local REPLACED + REPLACED=$(ssh -n "$SERVER" "cd ~/netork && \ + REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ + docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -aq ${SERVICES} \ + | xargs -r docker inspect --format '{{.Image}}' | sort -u" 2>/dev/null || true) + echo "[${SERVER}] Starting containers..." local RECREATE="cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ @@ -527,8 +535,17 @@ REMOTE_ENV run_remote "$SERVER" "alembic upgrade head (containers are ALREADY swapped)" "" \ "cd ~/netork && docker compose exec -T netork-api alembic upgrade head" - echo "[${SERVER}] Pruning unused images..." - ssh -n "$SERVER" "docker image prune -af 2>&1 | tail -1" | sed "s/^/[${SERVER}] /" + # The images the swap replaced, and nothing else. This was `docker image prune + # -af`, which removes every image no container uses at that moment: on a host + # that runs other things too, their rollback images (NetOrk/kb-server#157). + # remote/remove_replaced_images.sh keeps any image a container still uses. + if [[ -n "$REPLACED" ]]; then + echo "[${SERVER}] Removing the images this deploy replaced..." + local ids="${REPLACED//$'\n'/ }" + # shellcheck disable=SC2029 # the image IDs are meant to expand here + ssh "$SERVER" "bash -s -- ${ids}" < "${HERE}/remote/remove_replaced_images.sh" 2>&1 \ + | sed "s/^/[${SERVER}] /" || true + fi echo "[${SERVER}] Done." } diff --git a/remote/remove_replaced_images.sh b/remote/remove_replaced_images.sh new file mode 100644 index 0000000..ae059a1 --- /dev/null +++ b/remote/remove_replaced_images.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# remove_replaced_images.sh ...: the images a deploy replaced, and +# nothing else. deploy.sh records the IDs netOrk's own containers use before the +# swap and runs this on the host after it. +# +# It used to be `docker image prune -af`, which removes every image no container +# uses at that moment. On a host that runs other things too, that took their +# rollback images (NetOrk/kb-server#157). An ID goes here only if no container -- +# netOrk's or anyone else's -- uses it: a redeploy of the same tag replaces +# nothing. It goes by every tag it carries, or by its ID once a moving tag such as +# latest-dev has left it without one. +set -u + +in_use=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' 2>/dev/null | sort -u) + +for id in "$@"; do + if printf '%s\n' "$in_use" | grep -qx -- "$id"; then + echo "kept ${id:7:12} (in use)" + continue + fi + tags=$(docker image inspect --format '{{range .RepoTags}}{{.}} {{end}}' "$id" 2>/dev/null) || continue + # shellcheck disable=SC2086 # the tags are separate words on purpose + if docker image rm -- ${tags:-$id} >/dev/null 2>&1; then + echo "removed ${tags:-$id}" + else + echo "kept ${id:7:12}" + fi +done +exit 0 diff --git a/tests/test_deploy_replaced_images.py b/tests/test_deploy_replaced_images.py new file mode 100644 index 0000000..2f0c8f3 --- /dev/null +++ b/tests/test_deploy_replaced_images.py @@ -0,0 +1,80 @@ +"""A deploy removes the images it replaced, and nothing else (NetOrk/kb-server#157). + +It ended with `docker image prune -af`: every image no container used at that +moment, on a host that may run other things too. Now it records the image IDs of +netOrk's own containers before the swap and, once the swap is verified and the +migration ran, hands them to remote/remove_replaced_images.sh. + +The remote side is a fake `ssh` that records every command and plays a host whose +netOrk containers run one image. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" + +_FAKE_SSH = """#!/bin/sh +for arg in "$@"; do cmd="$arg"; done +case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac +case "$cmd" in + *"bash -s"*) cat > "{state}/stdin-$(date +%s%N)" ;; + *) cat > /dev/null ;; +esac +printf '%s\\n----\\n' "$cmd" >> "{state}/commands" +case "$cmd" in *"xargs -r docker inspect"*) echo sha256:old ;; esac +exit 0 +""" + + +def _deploy(tmp_path: Path) -> tuple[subprocess.CompletedProcess[str], list[str]]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake = bin_dir / "ssh" + fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path))) + fake.chmod(0o755) + result = subprocess.run( + ["bash", str(DEPLOY), "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + commands = (tmp_path / "commands").read_text().split("\n----\n") + return result, commands + + +def _index(commands: list[str], needle: str) -> int: + return next(i for i, cmd in enumerate(commands) if needle in cmd) + + +def test_no_prune_of_everything(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path) + assert result.returncode == 0, result.stderr + assert not [cmd for cmd in commands if "image prune -af" in cmd] + + +def test_the_replaced_images_are_recorded_before_the_swap_and_removed_after( + tmp_path: Path, +) -> None: + result, commands = _deploy(tmp_path) + assert result.returncode == 0, result.stderr + recorded = _index(commands, "xargs -r docker inspect") + swap = _index(commands, "--force-recreate") + migrate = _index(commands, "alembic upgrade head") + removal = _index(commands, "sha256:old") + assert recorded < swap < migrate < removal + assert "bash -s" in commands[removal] + sent = [p.read_text() for p in tmp_path.glob("stdin-*")] + assert any("remove_replaced_images.sh" in text for text in sent), result.stdout diff --git a/tests/test_remote_replaced_images.py b/tests/test_remote_replaced_images.py new file mode 100644 index 0000000..eb0136c --- /dev/null +++ b/tests/test_remote_replaced_images.py @@ -0,0 +1,120 @@ +"""After a deploy, the images it replaced go, and nothing else: remote/remove_replaced_images.sh. + +deploy.sh used to end with `docker image prune -af`, which removes every image no +container uses at that moment. On a host that runs other things too, that took +their rollback images (NetOrk/kb-server#157, the same line in the KB's script). + +deploy.sh now records the image IDs netOrk's own containers use before the swap +and hands them to this script afterwards. It removes those, by every tag they +carry, unless a container -- netOrk's or anyone else's -- still uses them: a +redeploy of the same tag replaces nothing. + +The remote side is a fake `docker` that keeps images and containers in files. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "remove_replaced_images.sh" + +_FAKE_DOCKER = r"""#!/bin/bash +state=$FAKE_STATE +printf '%s\n' "$*" >> "$state/calls" +case "$1 $2" in + "ps -aq") cut -d' ' -f1 "$state/containers" ;; + "inspect --format") + shift 3 + for c in "$@"; do grep "^$c " "$state/containers" | cut -d' ' -f2; done ;; + "image inspect") + id="${*: -1}" + line=$(grep "^$id" "$state/images") || exit 1 + echo "${line#* }" ;; + "image rm") + shift 2 + [ "$1" = -- ] && shift + for ref in "$@"; do + if grep -q " $ref\b" "$state/containers" 2>/dev/null; then exit 1; fi + sed -i "s# $ref##" "$state/images" + done + # An image without tags is gone. + sed -i '/^[^ ]*$/d' "$state/images" ;; +esac +exit 0 +""" + + +def _run(tmp_path: Path, images: list[str], containers: list[str], ids: list[str]): + (tmp_path / "images").write_text("".join(f"{line}\n" for line in images)) + (tmp_path / "containers").write_text("".join(f"{line}\n" for line in containers)) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + (bin_dir / "docker").write_text(_FAKE_DOCKER) + (bin_dir / "docker").chmod(0o755) + result = subprocess.run( + ["bash", str(SCRIPT), *ids], + env={"PATH": f"{bin_dir}:{os.environ['PATH']}", "FAKE_STATE": str(tmp_path)}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + left = (tmp_path / "images").read_text().split() + return result, left + + +def test_the_replaced_release_goes_by_every_tag(tmp_path: Path) -> None: + result, left = _run( + tmp_path, + images=[ + "sha256:old r/netork/engine:0.41.0 r/netork/engine:main-aaa", + "sha256:new r/netork/engine:0.42.0", + ], + containers=["c1 sha256:new"], + ids=["sha256:old"], + ) + assert result.returncode == 0, result.stderr + assert "sha256:old" not in left + assert "sha256:new" in left + assert "removed" in result.stdout + + +def test_an_image_still_in_use_stays(tmp_path: Path) -> None: + # A redeploy of the same tag: the "old" image is the one running now. + result, left = _run( + tmp_path, + images=["sha256:same r/netork/engine:latest-dev r/netork/engine:main-bbb"], + containers=["c1 sha256:same"], + ids=["sha256:same"], + ) + assert result.returncode == 0 + assert left == ["sha256:same", "r/netork/engine:latest-dev", "r/netork/engine:main-bbb"] + assert "kept" in result.stdout + + +def test_another_projects_images_are_never_named(tmp_path: Path) -> None: + result, left = _run( + tmp_path, + images=["sha256:old r/netork/engine:0.41.0", "sha256:theirs other/app:1.0"], + containers=[], + ids=["sha256:old"], + ) + assert result.returncode == 0 + assert "sha256:theirs" in left + assert "prune" not in (tmp_path / "calls").read_text() + + +def test_an_untagged_leftover_goes_by_its_id(tmp_path: Path) -> None: + # latest-dev moved on: the old image has no tag left. + result, left = _run( + tmp_path, images=["sha256:dangling"], containers=[], ids=["sha256:dangling"] + ) + assert result.returncode == 0 + assert "sha256:dangling" not in left + + +def test_nothing_to_remove_is_no_failure(tmp_path: Path) -> None: + result, _ = _run(tmp_path, images=[], containers=[], ids=["sha256:gone"]) + assert result.returncode == 0 -- 2.54.0