From 7e4fff8025ce63775afce5d78099c204a217d189 Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 20 Oct 2026 22:26:25 +0200 Subject: [PATCH] feat: reconcile the broker before the engine (NetOrk/netork#1308) redis was in no list: the engine is recreated with --no-deps, and the infrastructure reconcile left databases out on purpose. A changed definition -- Valkey in place of redis:7-alpine -- therefore never reached a host. The broker is now brought up with a plain `up -d --wait` right before the engine is recreated: recreated only when its definition changed, so an unchanged broker keeps running and keeps its queue, and the engine comes up against a healthy broker. If the broker does not get healthy within 120 s, the deploy stops with the old engine still running. --- deploy.sh | 19 ++++++- tests/test_deploy_broker.py | 82 ++++++++++++++++++++++++++++++ tests/test_deploy_env_before_up.py | 6 +-- 3 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 tests/test_deploy_broker.py diff --git a/deploy.sh b/deploy.sh index 0861688..cad1183 100644 --- a/deploy.sh +++ b/deploy.sh @@ -306,8 +306,9 @@ deploy_steps() { fi # Containers on upstream images. Reconciled, not force-recreated (below). - # postgres and redis are deliberately absent: restarting a database on every - # deploy would be worse than any compose change one could miss. + # postgres is deliberately absent: restarting a database on every deploy would + # be worse than any compose change one could miss. redis is reconciled on its + # own, before the engine (below). local INFRA_SERVICES="apt-cacher-ng signal-api" echo "[${SERVER}] Pulling images..." @@ -352,6 +353,20 @@ for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; d done REMOTE_ENV + # The broker, before the engine that cannot work without it. Plain `up -d` + # recreates it only when its compose definition changed, so an unchanged + # broker keeps running and keeps its queue. It used to be in no list at all, + # and a changed definition -- Valkey in place of redis:7-alpine + # (NetOrk/netork#1308) -- never reached a host. --wait: the engine comes up + # against a healthy broker. If it does not get there, the deploy stops here, + # with the old engine still running. + echo "[${SERVER}] Reconciling the broker..." + run_remote "$SERVER" "reconciling redis" "" \ + "cd ~/netork && \ + REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ + docker compose -f docker-compose.yml -f docker-compose.registry.yml \ + up -d --no-deps --wait --wait-timeout 120 redis" + # --force-recreate is deliberate: `docker compose up -d` decides what to # replace from the service definition's config hash, and the image reference # (`.../ui:latest-dev`) does not change when the tag is moved to a new image. diff --git a/tests/test_deploy_broker.py b/tests/test_deploy_broker.py new file mode 100644 index 0000000..960d305 --- /dev/null +++ b/tests/test_deploy_broker.py @@ -0,0 +1,82 @@ +"""The broker is reconciled before the engine is recreated (NetOrk/netork#1308). + +`redis` was in neither list: the engine is recreated with --no-deps, and the +infrastructure reconcile left databases out on purpose. A change to its compose +definition -- Valkey in place of redis:7-alpine -- therefore never reached a +host. A plain `up -d` recreates it only when its definition changed, so an +unchanged broker keeps running. It comes before the engine, which cannot work +without it, and waits until it is healthy; when it does not come up, the deploy +stops there. + +The remote side is a fake `ssh` that records every command it is given. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" + +_FAKE_SSH = """#!/bin/sh +for arg in "$@"; do cmd="$arg"; done +case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac +cat > /dev/null +printf '%s\\n----\\n' "$cmd" >> "{state}/commands" +case "$cmd" in *"up -d --no-deps --wait"*redis*) exit {broker_exit} ;; esac +exit 0 +""" + + +def _deploy(tmp_path: Path, broker_exit: int = 0) -> tuple[subprocess.CompletedProcess[str], list[str]]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake = bin_dir / "ssh" + fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{broker_exit}", str(broker_exit))) + fake.chmod(0o755) + result = subprocess.run( + ["bash", str(DEPLOY), "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + commands = (tmp_path / "commands").read_text().split("\n----\n") + return result, commands + + +def _index(commands: list[str], needle: str) -> int: + return next(i for i, c in enumerate(commands) if needle in c) + + +def test_the_broker_is_reconciled_before_the_engine(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path) + + assert result.returncode == 0, result.stderr + broker = _index(commands, "up -d --no-deps --wait --wait-timeout 120 redis") + engine = _index(commands, "up -d --no-deps --force-recreate") + assert broker < engine + + +def test_the_broker_is_never_force_recreated(tmp_path: Path) -> None: + _, commands = _deploy(tmp_path) + + for c in commands: + if "--force-recreate" in c: + assert " redis" not in c.replace("--no-deps", "") + + +def test_a_broker_that_does_not_come_up_stops_the_deploy(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path, broker_exit=1) + + assert result.returncode != 0 + assert not any("--force-recreate" in c for c in commands) diff --git a/tests/test_deploy_env_before_up.py b/tests/test_deploy_env_before_up.py index 2d0fb83..7ab973e 100644 --- a/tests/test_deploy_env_before_up.py +++ b/tests/test_deploy_env_before_up.py @@ -76,9 +76,9 @@ def test_every_up_sees_the_version_being_deployed(tmp_path: Path) -> None: assert result.returncode == 0, result.stderr seen = (tmp_path / "env-at-up").read_text().splitlines() - # The engine services' recreate, then the infrastructure reconcile. - assert len(seen) == 2, seen - assert seen == ["NETORK_VERSION=latest-dev"] * 2, ( + # The broker, the engine services' recreate, then the infrastructure reconcile. + assert len(seen) == 3, seen + assert seen == ["NETORK_VERSION=latest-dev"] * 3, ( "a container was started while .env still named the previous deploy's version" )