fix: hand ssl_certs to netork before the swap (NetOrk/netork#1389) #15
No files matched your search
@@ -108,20 +108,25 @@ started earlier pulls whatever image the registry held before, which is stale.
|
|||||||
6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed,
|
6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed,
|
||||||
and waited for until healthy, before the engine comes up against it. If it does
|
and waited for until healthy, before the engine comes up against it. If it does
|
||||||
not get healthy within 120 s, the deploy stops with the old engine running.
|
not get healthy within 120 s, the deploy stops with the old engine running.
|
||||||
7. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers,
|
7. Hands the `ssl_certs` volume to `netork`, the user the API and nginx run as
|
||||||
|
(NetOrk/netork#1389), in one container of the engine image run as root with only the
|
||||||
|
capability to chown. A volume written by the old root nginx would keep the UI from
|
||||||
|
starting. `chown -c` names only what it changed; on a host handed over earlier it
|
||||||
|
does nothing. If it fails, the deploy stops with the old containers running.
|
||||||
|
8. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers,
|
||||||
`netork-ui`. If the recreate fails, it is tried once more after 5 s
|
`netork-ui`. If the recreate fails, it is tried once more after 5 s
|
||||||
(`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it
|
(`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it
|
||||||
just renamed and leave the rest stopped. If the second attempt fails too, the
|
just renamed and leave the rest stopped. If the second attempt fails too, the
|
||||||
container states are printed and the deploy fails.
|
container states are printed and the deploy fails.
|
||||||
8. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
|
9. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
|
||||||
definition changed.
|
definition changed.
|
||||||
It then removes services netOrk no longer ships, container and volume, where a host
|
It then removes services netOrk no longer ships, container and volume, where a host
|
||||||
still has them: today the bundled `registry` (NetOrk/netork#763). It reports only
|
still has them: today the bundled `registry` (NetOrk/netork#763). It reports only
|
||||||
what it actually removed; on a host that was cleaned earlier it says nothing.
|
what it actually removed; on a host that was cleaned earlier it says nothing.
|
||||||
9. Verifies that the containers run exactly the image that was pulled. If they don't, the
|
10. Verifies that the containers run exactly the image that was pulled. If they don't, the
|
||||||
deploy fails.
|
deploy fails.
|
||||||
10. Runs `alembic upgrade head` inside `netork-api`.
|
11. Runs `alembic upgrade head` inside `netork-api`.
|
||||||
11. Prunes unused images.
|
12. Prunes unused images.
|
||||||
|
|
||||||
A failing step stops that server's deploy with a non-zero exit, and the other servers
|
A failing step stops that server's deploy with a non-zero exit, and the other servers
|
||||||
carry on. The script exits non-zero if any server failed and names those servers.
|
carry on. The script exits non-zero if any server failed and names those servers.
|
||||||
|
|||||||
@@ -399,6 +399,22 @@ REMOTE_ENV
|
|||||||
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
|
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
|
||||||
up -d --no-deps --wait --wait-timeout 120 redis"
|
up -d --no-deps --wait --wait-timeout 120 redis"
|
||||||
|
|
||||||
|
# ssl_certs to netork, before the swap (NetOrk/netork#1388, #1389). netork-ui
|
||||||
|
# ran nginx as root, and its entrypoint wrote the self-signed certificate into
|
||||||
|
# this volume as root, the key at 0600. nginx now runs as netork, the API's
|
||||||
|
# user, and cannot read that key: the UI would not start. One container of the
|
||||||
|
# engine image of this version, as root with only the capability to chown,
|
||||||
|
# hands the volume over. `chown -c` names only what it changed, so once a host
|
||||||
|
# is handed over this does nothing and says nothing. If it fails, the deploy
|
||||||
|
# stops here, with the old containers running.
|
||||||
|
echo "[${SERVER}] Handing ssl_certs to netork..."
|
||||||
|
run_remote "$SERVER" "handing ssl_certs to netork" "" \
|
||||||
|
"cd ~/netork && \
|
||||||
|
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
|
||||||
|
run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api \
|
||||||
|
-Rc netork:netork /app/ssl"
|
||||||
|
|
||||||
# --force-recreate is deliberate: `docker compose up -d` decides what to
|
# --force-recreate is deliberate: `docker compose up -d` decides what to
|
||||||
# replace from the service definition's config hash, and the image reference
|
# replace from the service definition's config hash, and the image reference
|
||||||
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
|
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""The ssl_certs volume is handed to netork before the containers are recreated
|
||||||
|
(NetOrk/netork#1388, #1389).
|
||||||
|
|
||||||
|
netork-ui ran nginx as root, and its entrypoint wrote the self-signed
|
||||||
|
certificate into ssl_certs as root, the key at 0600. From netork#1389 on, nginx
|
||||||
|
runs as netork (uid 1000, the API's user), and cannot read a key root wrote:
|
||||||
|
the UI would not start. One container of the engine image, as root with only
|
||||||
|
the capability to chown, hands the volume over before the swap. `chown -c`
|
||||||
|
reports only what it changed, so on a host already handed over it does
|
||||||
|
nothing and says nothing. Should it fail, the deploy stops with the old
|
||||||
|
containers still running.
|
||||||
|
|
||||||
|
The remote side is a fake `ssh` that records every command it is given.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
|
||||||
|
HAND_OVER = (
|
||||||
|
"run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api "
|
||||||
|
"-Rc netork:netork /app/ssl"
|
||||||
|
)
|
||||||
|
|
||||||
|
_FAKE_SSH = """#!/bin/sh
|
||||||
|
for arg in "$@"; do cmd="$arg"; done
|
||||||
|
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
|
||||||
|
cat > /dev/null
|
||||||
|
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
|
||||||
|
case "$cmd" in *"--entrypoint chown netork-api"*) exit {chown_exit} ;; esac
|
||||||
|
exit 0
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _deploy(
|
||||||
|
tmp_path: Path, chown_exit: int = 0
|
||||||
|
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
|
||||||
|
bin_dir = tmp_path / "bin"
|
||||||
|
bin_dir.mkdir()
|
||||||
|
fake = bin_dir / "ssh"
|
||||||
|
fake.write_text(
|
||||||
|
_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{chown_exit}", str(chown_exit))
|
||||||
|
)
|
||||||
|
fake.chmod(0o755)
|
||||||
|
result = subprocess.run(
|
||||||
|
["bash", str(DEPLOY), "testhost"],
|
||||||
|
env={
|
||||||
|
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||||
|
"HOME": str(tmp_path),
|
||||||
|
"DEPLOY_ENV_FILE": "/dev/null",
|
||||||
|
"REGISTRY_HOST": "registry.example",
|
||||||
|
"NETORK_VERSION": "latest-dev",
|
||||||
|
},
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
# Whitespace as the remote shell splits it: continuation lines leave runs of blanks.
|
||||||
|
commands = [" ".join(c.split()) for c in (tmp_path / "commands").read_text().split("\n----\n")]
|
||||||
|
return result, commands
|
||||||
|
|
||||||
|
|
||||||
|
def _index(commands: list[str], needle: str) -> int:
|
||||||
|
return next(i for i, c in enumerate(commands) if needle in c)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_volume_is_handed_over_after_the_pull_and_before_the_swap(tmp_path: Path) -> None:
|
||||||
|
result, commands = _deploy(tmp_path)
|
||||||
|
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
pull = _index(commands, "pull netork-api")
|
||||||
|
hand_over = _index(commands, HAND_OVER)
|
||||||
|
swap = _index(commands, "up -d --no-deps --force-recreate")
|
||||||
|
assert pull < hand_over < swap
|
||||||
|
|
||||||
|
|
||||||
|
def test_it_runs_the_image_of_the_version_being_deployed(tmp_path: Path) -> None:
|
||||||
|
_, commands = _deploy(tmp_path)
|
||||||
|
|
||||||
|
command = commands[_index(commands, HAND_OVER)]
|
||||||
|
assert "NETORK_VERSION='latest-dev'" in command
|
||||||
|
assert "-f docker-compose.yml -f docker-compose.registry.yml" in command
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_failed_hand_over_stops_before_the_swap(tmp_path: Path) -> None:
|
||||||
|
result, commands = _deploy(tmp_path, chown_exit=1)
|
||||||
|
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert not any("--force-recreate" in c for c in commands)
|
||||||
|
assert "handing ssl_certs to netork" in result.stderr
|
||||||
Reference in new issue
Block a user