fix: remove only the images the deploy replaced, never every unused one #16

Open
christianmanivong wants to merge 1 commits from fix/remove-only-replaced-images into main
pull from: fix/remove-only-replaced-images
5 changed files with 254 additions and 3 deletions

No files matched your search

+6 -1
View File
@@ -126,7 +126,12 @@ started earlier pulls whatever image the registry held before, which is stale.
10. Verifies that the containers run exactly the image that was pulled. If they don't, the
deploy fails.
11. Runs `alembic upgrade head` inside `netork-api`.
12. Prunes unused images.
12. Removes the images the swap replaced, and only those. Before the swap it records which
images netOrk's own containers use; afterwards `remote/remove_replaced_images.sh`
removes them unless a container still uses one, a redeploy of the same tag for
example. Other projects' images on the host stay. This used to be
`docker image prune -af`, which took every unused image, theirs included
(NetOrk/kb-server#157).
A failing step stops that server's deploy with a non-zero exit, and the other servers
carry on. The script exits non-zero if any server failed and names those servers.
+19 -2
View File
@@ -429,6 +429,14 @@ REMOTE_ENV
# netOrk was down until someone ran the deploy again, which then went through
# (NetOrk/netork#586). The second attempt is that rerun. Should it fail as well,
# the container states go to the log and the deploy fails as before.
# The images netOrk's own containers use now, before the swap: after it, these
# are what the deploy replaced, and the only images it removes (below).
local REPLACED
REPLACED=$(ssh -n "$SERVER" "cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -aq ${SERVICES} \
| xargs -r docker inspect --format '{{.Image}}' | sort -u" 2>/dev/null || true)
echo "[${SERVER}] Starting containers..."
local RECREATE="cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
@@ -527,8 +535,17 @@ REMOTE_ENV
run_remote "$SERVER" "alembic upgrade head (containers are ALREADY swapped)" "" \
"cd ~/netork && docker compose exec -T netork-api alembic upgrade head"
echo "[${SERVER}] Pruning unused images..."
ssh -n "$SERVER" "docker image prune -af 2>&1 | tail -1" | sed "s/^/[${SERVER}] /"
# The images the swap replaced, and nothing else. This was `docker image prune
# -af`, which removes every image no container uses at that moment: on a host
# that runs other things too, their rollback images (NetOrk/kb-server#157).
# remote/remove_replaced_images.sh keeps any image a container still uses.
if [[ -n "$REPLACED" ]]; then
echo "[${SERVER}] Removing the images this deploy replaced..."
local ids="${REPLACED//$'\n'/ }"
# shellcheck disable=SC2029 # the image IDs are meant to expand here
ssh "$SERVER" "bash -s -- ${ids}" < "${HERE}/remote/remove_replaced_images.sh" 2>&1 \
| sed "s/^/[${SERVER}] /" || true
fi
echo "[${SERVER}] Done."
}
+29
View File
@@ -0,0 +1,29 @@
#!/bin/bash
# remove_replaced_images.sh <image id>...: the images a deploy replaced, and
# nothing else. deploy.sh records the IDs netOrk's own containers use before the
# swap and runs this on the host after it.
#
# It used to be `docker image prune -af`, which removes every image no container
# uses at that moment. On a host that runs other things too, that took their
# rollback images (NetOrk/kb-server#157). An ID goes here only if no container --
# netOrk's or anyone else's -- uses it: a redeploy of the same tag replaces
# nothing. It goes by every tag it carries, or by its ID once a moving tag such as
# latest-dev has left it without one.
set -u
in_use=$(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' 2>/dev/null | sort -u)
for id in "$@"; do
if printf '%s\n' "$in_use" | grep -qx -- "$id"; then
echo "kept ${id:7:12} (in use)"
continue
fi
tags=$(docker image inspect --format '{{range .RepoTags}}{{.}} {{end}}' "$id" 2>/dev/null) || continue
# shellcheck disable=SC2086 # the tags are separate words on purpose
if docker image rm -- ${tags:-$id} >/dev/null 2>&1; then
echo "removed ${tags:-$id}"
else
echo "kept ${id:7:12}"
fi
done
exit 0
+80
View File
@@ -0,0 +1,80 @@
"""A deploy removes the images it replaced, and nothing else (NetOrk/kb-server#157).
It ended with `docker image prune -af`: every image no container used at that
moment, on a host that may run other things too. Now it records the image IDs of
netOrk's own containers before the swap and, once the swap is verified and the
migration ran, hands them to remote/remove_replaced_images.sh.
The remote side is a fake `ssh` that records every command and plays a host whose
netOrk containers run one image.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
_FAKE_SSH = """#!/bin/sh
for arg in "$@"; do cmd="$arg"; done
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
case "$cmd" in
*"bash -s"*) cat > "{state}/stdin-$(date +%s%N)" ;;
*) cat > /dev/null ;;
esac
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
case "$cmd" in *"xargs -r docker inspect"*) echo sha256:old ;; esac
exit 0
"""
def _deploy(tmp_path: Path) -> tuple[subprocess.CompletedProcess[str], list[str]]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
fake = bin_dir / "ssh"
fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)))
fake.chmod(0o755)
result = subprocess.run(
["bash", str(DEPLOY), "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
"NETORK_VERSION": "latest-dev",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=60,
check=False,
)
commands = (tmp_path / "commands").read_text().split("\n----\n")
return result, commands
def _index(commands: list[str], needle: str) -> int:
return next(i for i, cmd in enumerate(commands) if needle in cmd)
def test_no_prune_of_everything(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
assert not [cmd for cmd in commands if "image prune -af" in cmd]
def test_the_replaced_images_are_recorded_before_the_swap_and_removed_after(
tmp_path: Path,
) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
recorded = _index(commands, "xargs -r docker inspect")
swap = _index(commands, "--force-recreate")
migrate = _index(commands, "alembic upgrade head")
removal = _index(commands, "sha256:old")
assert recorded < swap < migrate < removal
assert "bash -s" in commands[removal]
sent = [p.read_text() for p in tmp_path.glob("stdin-*")]
assert any("remove_replaced_images.sh" in text for text in sent), result.stdout
+120
View File
@@ -0,0 +1,120 @@
"""After a deploy, the images it replaced go, and nothing else: remote/remove_replaced_images.sh.
deploy.sh used to end with `docker image prune -af`, which removes every image no
container uses at that moment. On a host that runs other things too, that took
their rollback images (NetOrk/kb-server#157, the same line in the KB's script).
deploy.sh now records the image IDs netOrk's own containers use before the swap
and hands them to this script afterwards. It removes those, by every tag they
carry, unless a container -- netOrk's or anyone else's -- still uses them: a
redeploy of the same tag replaces nothing.
The remote side is a fake `docker` that keeps images and containers in files.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "remove_replaced_images.sh"
_FAKE_DOCKER = r"""#!/bin/bash
state=$FAKE_STATE
printf '%s\n' "$*" >> "$state/calls"
case "$1 $2" in
"ps -aq") cut -d' ' -f1 "$state/containers" ;;
"inspect --format")
shift 3
for c in "$@"; do grep "^$c " "$state/containers" | cut -d' ' -f2; done ;;
"image inspect")
id="${*: -1}"
line=$(grep "^$id" "$state/images") || exit 1
echo "${line#* }" ;;
"image rm")
shift 2
[ "$1" = -- ] && shift
for ref in "$@"; do
if grep -q " $ref\b" "$state/containers" 2>/dev/null; then exit 1; fi
sed -i "s# $ref##" "$state/images"
done
# An image without tags is gone.
sed -i '/^[^ ]*$/d' "$state/images" ;;
esac
exit 0
"""
def _run(tmp_path: Path, images: list[str], containers: list[str], ids: list[str]):
(tmp_path / "images").write_text("".join(f"{line}\n" for line in images))
(tmp_path / "containers").write_text("".join(f"{line}\n" for line in containers))
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
(bin_dir / "docker").write_text(_FAKE_DOCKER)
(bin_dir / "docker").chmod(0o755)
result = subprocess.run(
["bash", str(SCRIPT), *ids],
env={"PATH": f"{bin_dir}:{os.environ['PATH']}", "FAKE_STATE": str(tmp_path)},
capture_output=True,
text=True,
timeout=30,
check=False,
)
left = (tmp_path / "images").read_text().split()
return result, left
def test_the_replaced_release_goes_by_every_tag(tmp_path: Path) -> None:
result, left = _run(
tmp_path,
images=[
"sha256:old r/netork/engine:0.41.0 r/netork/engine:main-aaa",
"sha256:new r/netork/engine:0.42.0",
],
containers=["c1 sha256:new"],
ids=["sha256:old"],
)
assert result.returncode == 0, result.stderr
assert "sha256:old" not in left
assert "sha256:new" in left
assert "removed" in result.stdout
def test_an_image_still_in_use_stays(tmp_path: Path) -> None:
# A redeploy of the same tag: the "old" image is the one running now.
result, left = _run(
tmp_path,
images=["sha256:same r/netork/engine:latest-dev r/netork/engine:main-bbb"],
containers=["c1 sha256:same"],
ids=["sha256:same"],
)
assert result.returncode == 0
assert left == ["sha256:same", "r/netork/engine:latest-dev", "r/netork/engine:main-bbb"]
assert "kept" in result.stdout
def test_another_projects_images_are_never_named(tmp_path: Path) -> None:
result, left = _run(
tmp_path,
images=["sha256:old r/netork/engine:0.41.0", "sha256:theirs other/app:1.0"],
containers=[],
ids=["sha256:old"],
)
assert result.returncode == 0
assert "sha256:theirs" in left
assert "prune" not in (tmp_path / "calls").read_text()
def test_an_untagged_leftover_goes_by_its_id(tmp_path: Path) -> None:
# latest-dev moved on: the old image has no tag left.
result, left = _run(
tmp_path, images=["sha256:dangling"], containers=[], ids=["sha256:dangling"]
)
assert result.returncode == 0
assert "sha256:dangling" not in left
def test_nothing_to_remove_is_no_failure(tmp_path: Path) -> None:
result, _ = _run(tmp_path, images=[], containers=[], ids=["sha256:gone"])
assert result.returncode == 0