fix: remove only the images the deploy replaced, never every unused one
CI / check (pull_request) Successful in 19s
CI / check (pull_request) Successful in 19s
The deploy ended with docker image prune -af: every image no container used at that moment. On a host that runs other things too, that takes their rollback images; the KB's copy of this line did exactly that on compute-01 (NetOrk/kb-server#157). deploy.sh now records the image IDs netOrk's own containers use before the swap and, after the verification and the migration, hands them to remote/remove_replaced_images.sh. It removes each by its tags, or by its ID once a moving tag left it without one, and keeps any image a container still uses: a redeploy of the same tag replaces nothing.
This commit is contained in:
1 parent
e35d42a20b
commit
476afe3982
5 files changed
+254
-3
No files matched your search
@@ -0,0 +1,80 @@
|
||||
"""A deploy removes the images it replaced, and nothing else (NetOrk/kb-server#157).
|
||||
|
||||
It ended with `docker image prune -af`: every image no container used at that
|
||||
moment, on a host that may run other things too. Now it records the image IDs of
|
||||
netOrk's own containers before the swap and, once the swap is verified and the
|
||||
migration ran, hands them to remote/remove_replaced_images.sh.
|
||||
|
||||
The remote side is a fake `ssh` that records every command and plays a host whose
|
||||
netOrk containers run one image.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
|
||||
|
||||
_FAKE_SSH = """#!/bin/sh
|
||||
for arg in "$@"; do cmd="$arg"; done
|
||||
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
|
||||
case "$cmd" in
|
||||
*"bash -s"*) cat > "{state}/stdin-$(date +%s%N)" ;;
|
||||
*) cat > /dev/null ;;
|
||||
esac
|
||||
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
|
||||
case "$cmd" in *"xargs -r docker inspect"*) echo sha256:old ;; esac
|
||||
exit 0
|
||||
"""
|
||||
|
||||
|
||||
def _deploy(tmp_path: Path) -> tuple[subprocess.CompletedProcess[str], list[str]]:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
fake = bin_dir / "ssh"
|
||||
fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)))
|
||||
fake.chmod(0o755)
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOY), "testhost"],
|
||||
env={
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"HOME": str(tmp_path),
|
||||
"DEPLOY_ENV_FILE": "/dev/null",
|
||||
"REGISTRY_HOST": "registry.example",
|
||||
"NETORK_VERSION": "latest-dev",
|
||||
},
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60,
|
||||
check=False,
|
||||
)
|
||||
commands = (tmp_path / "commands").read_text().split("\n----\n")
|
||||
return result, commands
|
||||
|
||||
|
||||
def _index(commands: list[str], needle: str) -> int:
|
||||
return next(i for i, cmd in enumerate(commands) if needle in cmd)
|
||||
|
||||
|
||||
def test_no_prune_of_everything(tmp_path: Path) -> None:
|
||||
result, commands = _deploy(tmp_path)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert not [cmd for cmd in commands if "image prune -af" in cmd]
|
||||
|
||||
|
||||
def test_the_replaced_images_are_recorded_before_the_swap_and_removed_after(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, commands = _deploy(tmp_path)
|
||||
assert result.returncode == 0, result.stderr
|
||||
recorded = _index(commands, "xargs -r docker inspect")
|
||||
swap = _index(commands, "--force-recreate")
|
||||
migrate = _index(commands, "alembic upgrade head")
|
||||
removal = _index(commands, "sha256:old")
|
||||
assert recorded < swap < migrate < removal
|
||||
assert "bash -s" in commands[removal]
|
||||
sent = [p.read_text() for p in tmp_path.glob("stdin-*")]
|
||||
assert any("remove_replaced_images.sh" in text for text in sent), result.stdout
|
||||
@@ -0,0 +1,120 @@
|
||||
"""After a deploy, the images it replaced go, and nothing else: remote/remove_replaced_images.sh.
|
||||
|
||||
deploy.sh used to end with `docker image prune -af`, which removes every image no
|
||||
container uses at that moment. On a host that runs other things too, that took
|
||||
their rollback images (NetOrk/kb-server#157, the same line in the KB's script).
|
||||
|
||||
deploy.sh now records the image IDs netOrk's own containers use before the swap
|
||||
and hands them to this script afterwards. It removes those, by every tag they
|
||||
carry, unless a container -- netOrk's or anyone else's -- still uses them: a
|
||||
redeploy of the same tag replaces nothing.
|
||||
|
||||
The remote side is a fake `docker` that keeps images and containers in files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "remove_replaced_images.sh"
|
||||
|
||||
_FAKE_DOCKER = r"""#!/bin/bash
|
||||
state=$FAKE_STATE
|
||||
printf '%s\n' "$*" >> "$state/calls"
|
||||
case "$1 $2" in
|
||||
"ps -aq") cut -d' ' -f1 "$state/containers" ;;
|
||||
"inspect --format")
|
||||
shift 3
|
||||
for c in "$@"; do grep "^$c " "$state/containers" | cut -d' ' -f2; done ;;
|
||||
"image inspect")
|
||||
id="${*: -1}"
|
||||
line=$(grep "^$id" "$state/images") || exit 1
|
||||
echo "${line#* }" ;;
|
||||
"image rm")
|
||||
shift 2
|
||||
[ "$1" = -- ] && shift
|
||||
for ref in "$@"; do
|
||||
if grep -q " $ref\b" "$state/containers" 2>/dev/null; then exit 1; fi
|
||||
sed -i "s# $ref##" "$state/images"
|
||||
done
|
||||
# An image without tags is gone.
|
||||
sed -i '/^[^ ]*$/d' "$state/images" ;;
|
||||
esac
|
||||
exit 0
|
||||
"""
|
||||
|
||||
|
||||
def _run(tmp_path: Path, images: list[str], containers: list[str], ids: list[str]):
|
||||
(tmp_path / "images").write_text("".join(f"{line}\n" for line in images))
|
||||
(tmp_path / "containers").write_text("".join(f"{line}\n" for line in containers))
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
(bin_dir / "docker").write_text(_FAKE_DOCKER)
|
||||
(bin_dir / "docker").chmod(0o755)
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), *ids],
|
||||
env={"PATH": f"{bin_dir}:{os.environ['PATH']}", "FAKE_STATE": str(tmp_path)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
left = (tmp_path / "images").read_text().split()
|
||||
return result, left
|
||||
|
||||
|
||||
def test_the_replaced_release_goes_by_every_tag(tmp_path: Path) -> None:
|
||||
result, left = _run(
|
||||
tmp_path,
|
||||
images=[
|
||||
"sha256:old r/netork/engine:0.41.0 r/netork/engine:main-aaa",
|
||||
"sha256:new r/netork/engine:0.42.0",
|
||||
],
|
||||
containers=["c1 sha256:new"],
|
||||
ids=["sha256:old"],
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "sha256:old" not in left
|
||||
assert "sha256:new" in left
|
||||
assert "removed" in result.stdout
|
||||
|
||||
|
||||
def test_an_image_still_in_use_stays(tmp_path: Path) -> None:
|
||||
# A redeploy of the same tag: the "old" image is the one running now.
|
||||
result, left = _run(
|
||||
tmp_path,
|
||||
images=["sha256:same r/netork/engine:latest-dev r/netork/engine:main-bbb"],
|
||||
containers=["c1 sha256:same"],
|
||||
ids=["sha256:same"],
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert left == ["sha256:same", "r/netork/engine:latest-dev", "r/netork/engine:main-bbb"]
|
||||
assert "kept" in result.stdout
|
||||
|
||||
|
||||
def test_another_projects_images_are_never_named(tmp_path: Path) -> None:
|
||||
result, left = _run(
|
||||
tmp_path,
|
||||
images=["sha256:old r/netork/engine:0.41.0", "sha256:theirs other/app:1.0"],
|
||||
containers=[],
|
||||
ids=["sha256:old"],
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "sha256:theirs" in left
|
||||
assert "prune" not in (tmp_path / "calls").read_text()
|
||||
|
||||
|
||||
def test_an_untagged_leftover_goes_by_its_id(tmp_path: Path) -> None:
|
||||
# latest-dev moved on: the old image has no tag left.
|
||||
result, left = _run(
|
||||
tmp_path, images=["sha256:dangling"], containers=[], ids=["sha256:dangling"]
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "sha256:dangling" not in left
|
||||
|
||||
|
||||
def test_nothing_to_remove_is_no_failure(tmp_path: Path) -> None:
|
||||
result, _ = _run(tmp_path, images=[], containers=[], ids=["sha256:gone"])
|
||||
assert result.returncode == 0
|
||||
Reference in new issue
Block a user