fix: remove only the images the deploy replaced, never every unused one
CI / check (pull_request) Successful in 19s

The deploy ended with docker image prune -af: every image no container
used at that moment. On a host that runs other things too, that takes
their rollback images; the KB's copy of this line did exactly that on
compute-01 (NetOrk/kb-server#157).

deploy.sh now records the image IDs netOrk's own containers use before
the swap and, after the verification and the migration, hands them to
remote/remove_replaced_images.sh. It removes each by its tags, or by its
ID once a moving tag left it without one, and keeps any image a
container still uses: a redeploy of the same tag replaces nothing.
This commit is contained in:
Christian Manivong committed 2026-10-23 12:06:08 +02:00
1 parent e35d42a20b
commit 476afe3982
5 files changed
+254 -3

No files matched your search

+80
View File
@@ -0,0 +1,80 @@
"""A deploy removes the images it replaced, and nothing else (NetOrk/kb-server#157).
It ended with `docker image prune -af`: every image no container used at that
moment, on a host that may run other things too. Now it records the image IDs of
netOrk's own containers before the swap and, once the swap is verified and the
migration ran, hands them to remote/remove_replaced_images.sh.
The remote side is a fake `ssh` that records every command and plays a host whose
netOrk containers run one image.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
_FAKE_SSH = """#!/bin/sh
for arg in "$@"; do cmd="$arg"; done
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
case "$cmd" in
*"bash -s"*) cat > "{state}/stdin-$(date +%s%N)" ;;
*) cat > /dev/null ;;
esac
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
case "$cmd" in *"xargs -r docker inspect"*) echo sha256:old ;; esac
exit 0
"""
def _deploy(tmp_path: Path) -> tuple[subprocess.CompletedProcess[str], list[str]]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
fake = bin_dir / "ssh"
fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)))
fake.chmod(0o755)
result = subprocess.run(
["bash", str(DEPLOY), "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
"NETORK_VERSION": "latest-dev",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=60,
check=False,
)
commands = (tmp_path / "commands").read_text().split("\n----\n")
return result, commands
def _index(commands: list[str], needle: str) -> int:
return next(i for i, cmd in enumerate(commands) if needle in cmd)
def test_no_prune_of_everything(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
assert not [cmd for cmd in commands if "image prune -af" in cmd]
def test_the_replaced_images_are_recorded_before_the_swap_and_removed_after(
tmp_path: Path,
) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
recorded = _index(commands, "xargs -r docker inspect")
swap = _index(commands, "--force-recreate")
migrate = _index(commands, "alembic upgrade head")
removal = _index(commands, "sha256:old")
assert recorded < swap < migrate < removal
assert "bash -s" in commands[removal]
sent = [p.read_text() for p in tmp_path.glob("stdin-*")]
assert any("remove_replaced_images.sh" in text for text in sent), result.stdout
+120
View File
@@ -0,0 +1,120 @@
"""After a deploy, the images it replaced go, and nothing else: remote/remove_replaced_images.sh.
deploy.sh used to end with `docker image prune -af`, which removes every image no
container uses at that moment. On a host that runs other things too, that took
their rollback images (NetOrk/kb-server#157, the same line in the KB's script).
deploy.sh now records the image IDs netOrk's own containers use before the swap
and hands them to this script afterwards. It removes those, by every tag they
carry, unless a container -- netOrk's or anyone else's -- still uses them: a
redeploy of the same tag replaces nothing.
The remote side is a fake `docker` that keeps images and containers in files.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "remove_replaced_images.sh"
_FAKE_DOCKER = r"""#!/bin/bash
state=$FAKE_STATE
printf '%s\n' "$*" >> "$state/calls"
case "$1 $2" in
"ps -aq") cut -d' ' -f1 "$state/containers" ;;
"inspect --format")
shift 3
for c in "$@"; do grep "^$c " "$state/containers" | cut -d' ' -f2; done ;;
"image inspect")
id="${*: -1}"
line=$(grep "^$id" "$state/images") || exit 1
echo "${line#* }" ;;
"image rm")
shift 2
[ "$1" = -- ] && shift
for ref in "$@"; do
if grep -q " $ref\b" "$state/containers" 2>/dev/null; then exit 1; fi
sed -i "s# $ref##" "$state/images"
done
# An image without tags is gone.
sed -i '/^[^ ]*$/d' "$state/images" ;;
esac
exit 0
"""
def _run(tmp_path: Path, images: list[str], containers: list[str], ids: list[str]):
(tmp_path / "images").write_text("".join(f"{line}\n" for line in images))
(tmp_path / "containers").write_text("".join(f"{line}\n" for line in containers))
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
(bin_dir / "docker").write_text(_FAKE_DOCKER)
(bin_dir / "docker").chmod(0o755)
result = subprocess.run(
["bash", str(SCRIPT), *ids],
env={"PATH": f"{bin_dir}:{os.environ['PATH']}", "FAKE_STATE": str(tmp_path)},
capture_output=True,
text=True,
timeout=30,
check=False,
)
left = (tmp_path / "images").read_text().split()
return result, left
def test_the_replaced_release_goes_by_every_tag(tmp_path: Path) -> None:
result, left = _run(
tmp_path,
images=[
"sha256:old r/netork/engine:0.41.0 r/netork/engine:main-aaa",
"sha256:new r/netork/engine:0.42.0",
],
containers=["c1 sha256:new"],
ids=["sha256:old"],
)
assert result.returncode == 0, result.stderr
assert "sha256:old" not in left
assert "sha256:new" in left
assert "removed" in result.stdout
def test_an_image_still_in_use_stays(tmp_path: Path) -> None:
# A redeploy of the same tag: the "old" image is the one running now.
result, left = _run(
tmp_path,
images=["sha256:same r/netork/engine:latest-dev r/netork/engine:main-bbb"],
containers=["c1 sha256:same"],
ids=["sha256:same"],
)
assert result.returncode == 0
assert left == ["sha256:same", "r/netork/engine:latest-dev", "r/netork/engine:main-bbb"]
assert "kept" in result.stdout
def test_another_projects_images_are_never_named(tmp_path: Path) -> None:
result, left = _run(
tmp_path,
images=["sha256:old r/netork/engine:0.41.0", "sha256:theirs other/app:1.0"],
containers=[],
ids=["sha256:old"],
)
assert result.returncode == 0
assert "sha256:theirs" in left
assert "prune" not in (tmp_path / "calls").read_text()
def test_an_untagged_leftover_goes_by_its_id(tmp_path: Path) -> None:
# latest-dev moved on: the old image has no tag left.
result, left = _run(
tmp_path, images=["sha256:dangling"], containers=[], ids=["sha256:dangling"]
)
assert result.returncode == 0
assert "sha256:dangling" not in left
def test_nothing_to_remove_is_no_failure(tmp_path: Path) -> None:
result, _ = _run(tmp_path, images=[], containers=[], ids=["sha256:gone"])
assert result.returncode == 0