feat: reconcile the broker before the engine (NetOrk/netork#1308)
CI / check (pull_request) Failing after 2m28s

redis was in no list: the engine is recreated with --no-deps, and the
infrastructure reconcile left databases out on purpose. A changed definition
-- Valkey in place of redis:7-alpine -- therefore never reached a host.

The broker is now brought up with a plain `up -d --wait` right before the
engine is recreated: recreated only when its definition changed, so an
unchanged broker keeps running and keeps its queue, and the engine comes up
against a healthy broker. If the broker does not get healthy within 120 s, the
deploy stops with the old engine still running.
This commit is contained in:
Christian Manivong committed 2026-10-20 22:26:25 +02:00
1 parent c1b1eb07b9
commit 7e4fff8025
3 files changed
+102 -5

No files matched your search

+82
View File
@@ -0,0 +1,82 @@
"""The broker is reconciled before the engine is recreated (NetOrk/netork#1308).
`redis` was in neither list: the engine is recreated with --no-deps, and the
infrastructure reconcile left databases out on purpose. A change to its compose
definition -- Valkey in place of redis:7-alpine -- therefore never reached a
host. A plain `up -d` recreates it only when its definition changed, so an
unchanged broker keeps running. It comes before the engine, which cannot work
without it, and waits until it is healthy; when it does not come up, the deploy
stops there.
The remote side is a fake `ssh` that records every command it is given.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
_FAKE_SSH = """#!/bin/sh
for arg in "$@"; do cmd="$arg"; done
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
cat > /dev/null
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
case "$cmd" in *"up -d --no-deps --wait"*redis*) exit {broker_exit} ;; esac
exit 0
"""
def _deploy(tmp_path: Path, broker_exit: int = 0) -> tuple[subprocess.CompletedProcess[str], list[str]]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
fake = bin_dir / "ssh"
fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{broker_exit}", str(broker_exit)))
fake.chmod(0o755)
result = subprocess.run(
["bash", str(DEPLOY), "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
"NETORK_VERSION": "latest-dev",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=60,
check=False,
)
commands = (tmp_path / "commands").read_text().split("\n----\n")
return result, commands
def _index(commands: list[str], needle: str) -> int:
return next(i for i, c in enumerate(commands) if needle in c)
def test_the_broker_is_reconciled_before_the_engine(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
broker = _index(commands, "up -d --no-deps --wait --wait-timeout 120 redis")
engine = _index(commands, "up -d --no-deps --force-recreate")
assert broker < engine
def test_the_broker_is_never_force_recreated(tmp_path: Path) -> None:
_, commands = _deploy(tmp_path)
for c in commands:
if "--force-recreate" in c:
assert " redis" not in c.replace("--no-deps", "")
def test_a_broker_that_does_not_come_up_stops_the_deploy(tmp_path: Path) -> None:
result, commands = _deploy(tmp_path, broker_exit=1)
assert result.returncode != 0
assert not any("--force-recreate" in c for c in commands)
+3 -3
View File
@@ -76,9 +76,9 @@ def test_every_up_sees_the_version_being_deployed(tmp_path: Path) -> None:
assert result.returncode == 0, result.stderr
seen = (tmp_path / "env-at-up").read_text().splitlines()
# The engine services' recreate, then the infrastructure reconcile.
assert len(seen) == 2, seen
assert seen == ["NETORK_VERSION=latest-dev"] * 2, (
# The broker, the engine services' recreate, then the infrastructure reconcile.
assert len(seen) == 3, seen
assert seen == ["NETORK_VERSION=latest-dev"] * 3, (
"a container was started while .env still named the previous deploy's version"
)