fix: hand ssl_certs to netork before the swap (NetOrk/netork#1389) #15

Merged
christianmanivong merged 1 commits from fix/ssl-certs-owner into main 2026-10-22 13:26:12 +00:00
3 changed files with 121 additions and 5 deletions
Showing only changes of commit 30da942a7a - Show all commits

No files matched your search

+10 -5
View File
@@ -108,20 +108,25 @@ started earlier pulls whatever image the registry held before, which is stale.
6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed,
and waited for until healthy, before the engine comes up against it. If it does
not get healthy within 120 s, the deploy stops with the old engine running.
7. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers,
7. Hands the `ssl_certs` volume to `netork`, the user the API and nginx run as
(NetOrk/netork#1389), in one container of the engine image run as root with only the
capability to chown. A volume written by the old root nginx would keep the UI from
starting. `chown -c` names only what it changed; on a host handed over earlier it
does nothing. If it fails, the deploy stops with the old containers running.
8. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers,
`netork-ui`. If the recreate fails, it is tried once more after 5 s
(`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it
just renamed and leave the rest stopped. If the second attempt fails too, the
container states are printed and the deploy fails.
8. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
9. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
definition changed.
It then removes services netOrk no longer ships, container and volume, where a host
still has them: today the bundled `registry` (NetOrk/netork#763). It reports only
what it actually removed; on a host that was cleaned earlier it says nothing.
9. Verifies that the containers run exactly the image that was pulled. If they don't, the
10. Verifies that the containers run exactly the image that was pulled. If they don't, the
deploy fails.
10. Runs `alembic upgrade head` inside `netork-api`.
11. Prunes unused images.
11. Runs `alembic upgrade head` inside `netork-api`.
12. Prunes unused images.
A failing step stops that server's deploy with a non-zero exit, and the other servers
carry on. The script exits non-zero if any server failed and names those servers.
+16
View File
@@ -399,6 +399,22 @@ REMOTE_ENV
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps --wait --wait-timeout 120 redis"
# ssl_certs to netork, before the swap (NetOrk/netork#1388, #1389). netork-ui
# ran nginx as root, and its entrypoint wrote the self-signed certificate into
# this volume as root, the key at 0600. nginx now runs as netork, the API's
# user, and cannot read that key: the UI would not start. One container of the
# engine image of this version, as root with only the capability to chown,
# hands the volume over. `chown -c` names only what it changed, so once a host
# is handed over this does nothing and says nothing. If it fails, the deploy
# stops here, with the old containers running.
echo "[${SERVER}] Handing ssl_certs to netork..."
run_remote "$SERVER" "handing ssl_certs to netork" "" \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api \
-Rc netork:netork /app/ssl"
# --force-recreate is deliberate: `docker compose up -d` decides what to
# replace from the service definition's config hash, and the image reference
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
+95
View File
@@ -0,0 +1,95 @@
"""The ssl_certs volume is handed to netork before the containers are recreated
(NetOrk/netork#1388, #1389).
netork-ui ran nginx as root, and its entrypoint wrote the self-signed
certificate into ssl_certs as root, the key at 0600. From netork#1389 on, nginx
runs as netork (uid 1000, the API's user), and cannot read a key root wrote:
the UI would not start. One container of the engine image, as root with only
the capability to chown, hands the volume over before the swap. `chown -c`
reports only what it changed, so on a host already handed over it does
nothing and says nothing. Should it fail, the deploy stops with the old
containers still running.
The remote side is a fake `ssh` that records every command it is given.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
HAND_OVER = (
"run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api "
"-Rc netork:netork /app/ssl"
)
_FAKE_SSH = """#!/bin/sh
for arg in "$@"; do cmd="$arg"; done
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
cat > /dev/null
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
case "$cmd" in *"--entrypoint chown netork-api"*) exit {chown_exit} ;; esac
exit 0
"""
def _deploy(
tmp_path: Path, chown_exit: int = 0
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
fake = bin_dir / "ssh"
fake.write_text(
_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{chown_exit}", str(chown_exit))
)
fake.chmod(0o755)
result = subprocess.run(
["bash", str(DEPLOY), "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
"NETORK_VERSION": "latest-dev",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=60,
check=False,
)
# Whitespace as the remote shell splits it: continuation lines leave runs of blanks.
commands = [" ".join(c.split()) for c in (tmp_path / "commands").read_text().split("\n----\n")]
return result, commands
def _index(commands: list[str], needle: str) -> int:
return next(i for i, c in enumerate(commands) if needle in c)
def test_the_volume_is_handed_over_after_the_pull_and_before_the_swap(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
pull = _index(commands, "pull netork-api")
hand_over = _index(commands, HAND_OVER)
swap = _index(commands, "up -d --no-deps --force-recreate")
assert pull < hand_over < swap
def test_it_runs_the_image_of_the_version_being_deployed(tmp_path: Path) -> None:
_, commands = _deploy(tmp_path)
command = commands[_index(commands, HAND_OVER)]
assert "NETORK_VERSION='latest-dev'" in command
assert "-f docker-compose.yml -f docker-compose.registry.yml" in command
def test_a_failed_hand_over_stops_before_the_swap(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path, chown_exit=1)
assert result.returncode != 0
assert not any("--force-recreate" in c for c in commands)
assert "handing ssl_certs to netork" in result.stderr