feat: reconcile the broker before the engine (NetOrk/netork#1308) #13
No files matched your search
@@ -306,8 +306,9 @@ deploy_steps() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Containers on upstream images. Reconciled, not force-recreated (below).
|
# Containers on upstream images. Reconciled, not force-recreated (below).
|
||||||
# postgres and redis are deliberately absent: restarting a database on every
|
# postgres is deliberately absent: restarting a database on every deploy would
|
||||||
# deploy would be worse than any compose change one could miss.
|
# be worse than any compose change one could miss. redis is reconciled on its
|
||||||
|
# own, before the engine (below).
|
||||||
local INFRA_SERVICES="apt-cacher-ng signal-api"
|
local INFRA_SERVICES="apt-cacher-ng signal-api"
|
||||||
|
|
||||||
echo "[${SERVER}] Pulling images..."
|
echo "[${SERVER}] Pulling images..."
|
||||||
@@ -352,6 +353,20 @@ for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; d
|
|||||||
done
|
done
|
||||||
REMOTE_ENV
|
REMOTE_ENV
|
||||||
|
|
||||||
|
# The broker, before the engine that cannot work without it. Plain `up -d`
|
||||||
|
# recreates it only when its compose definition changed, so an unchanged
|
||||||
|
# broker keeps running and keeps its queue. It used to be in no list at all,
|
||||||
|
# and a changed definition -- Valkey in place of redis:7-alpine
|
||||||
|
# (NetOrk/netork#1308) -- never reached a host. --wait: the engine comes up
|
||||||
|
# against a healthy broker. If it does not get there, the deploy stops here,
|
||||||
|
# with the old engine still running.
|
||||||
|
echo "[${SERVER}] Reconciling the broker..."
|
||||||
|
run_remote "$SERVER" "reconciling redis" "" \
|
||||||
|
"cd ~/netork && \
|
||||||
|
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
|
||||||
|
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
|
||||||
|
up -d --no-deps --wait --wait-timeout 120 redis"
|
||||||
|
|
||||||
# --force-recreate is deliberate: `docker compose up -d` decides what to
|
# --force-recreate is deliberate: `docker compose up -d` decides what to
|
||||||
# replace from the service definition's config hash, and the image reference
|
# replace from the service definition's config hash, and the image reference
|
||||||
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
|
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""The broker is reconciled before the engine is recreated (NetOrk/netork#1308).
|
||||||
|
|
||||||
|
`redis` was in neither list: the engine is recreated with --no-deps, and the
|
||||||
|
infrastructure reconcile left databases out on purpose. A change to its compose
|
||||||
|
definition -- Valkey in place of redis:7-alpine -- therefore never reached a
|
||||||
|
host. A plain `up -d` recreates it only when its definition changed, so an
|
||||||
|
unchanged broker keeps running. It comes before the engine, which cannot work
|
||||||
|
without it, and waits until it is healthy; when it does not come up, the deploy
|
||||||
|
stops there.
|
||||||
|
|
||||||
|
The remote side is a fake `ssh` that records every command it is given.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
|
||||||
|
|
||||||
|
_FAKE_SSH = """#!/bin/sh
|
||||||
|
for arg in "$@"; do cmd="$arg"; done
|
||||||
|
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
|
||||||
|
cat > /dev/null
|
||||||
|
printf '%s\\n----\\n' "$cmd" >> "{state}/commands"
|
||||||
|
case "$cmd" in *"up -d --no-deps --wait"*redis*) exit {broker_exit} ;; esac
|
||||||
|
exit 0
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _deploy(
|
||||||
|
tmp_path: Path, broker_exit: int = 0
|
||||||
|
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
|
||||||
|
bin_dir = tmp_path / "bin"
|
||||||
|
bin_dir.mkdir()
|
||||||
|
fake = bin_dir / "ssh"
|
||||||
|
fake.write_text(
|
||||||
|
_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{broker_exit}", str(broker_exit))
|
||||||
|
)
|
||||||
|
fake.chmod(0o755)
|
||||||
|
result = subprocess.run(
|
||||||
|
["bash", str(DEPLOY), "testhost"],
|
||||||
|
env={
|
||||||
|
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||||
|
"HOME": str(tmp_path),
|
||||||
|
"DEPLOY_ENV_FILE": "/dev/null",
|
||||||
|
"REGISTRY_HOST": "registry.example",
|
||||||
|
"NETORK_VERSION": "latest-dev",
|
||||||
|
},
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
commands = (tmp_path / "commands").read_text().split("\n----\n")
|
||||||
|
return result, commands
|
||||||
|
|
||||||
|
|
||||||
|
def _index(commands: list[str], needle: str) -> int:
|
||||||
|
return next(i for i, c in enumerate(commands) if needle in c)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_broker_is_reconciled_before_the_engine(tmp_path: Path) -> None:
|
||||||
|
result, commands = _deploy(tmp_path)
|
||||||
|
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
broker = _index(commands, "up -d --no-deps --wait --wait-timeout 120 redis")
|
||||||
|
engine = _index(commands, "up -d --no-deps --force-recreate")
|
||||||
|
assert broker < engine
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_broker_is_never_force_recreated(tmp_path: Path) -> None:
|
||||||
|
_, commands = _deploy(tmp_path)
|
||||||
|
|
||||||
|
for c in commands:
|
||||||
|
if "--force-recreate" in c:
|
||||||
|
assert " redis" not in c.replace("--no-deps", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_broker_that_does_not_come_up_stops_the_deploy(tmp_path: Path) -> None:
|
||||||
|
result, commands = _deploy(tmp_path, broker_exit=1)
|
||||||
|
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert not any("--force-recreate" in c for c in commands)
|
||||||
@@ -76,9 +76,9 @@ def test_every_up_sees_the_version_being_deployed(tmp_path: Path) -> None:
|
|||||||
|
|
||||||
assert result.returncode == 0, result.stderr
|
assert result.returncode == 0, result.stderr
|
||||||
seen = (tmp_path / "env-at-up").read_text().splitlines()
|
seen = (tmp_path / "env-at-up").read_text().splitlines()
|
||||||
# The engine services' recreate, then the infrastructure reconcile.
|
# The broker, the engine services' recreate, then the infrastructure reconcile.
|
||||||
assert len(seen) == 2, seen
|
assert len(seen) == 3, seen
|
||||||
assert seen == ["NETORK_VERSION=latest-dev"] * 2, (
|
assert seen == ["NETORK_VERSION=latest-dev"] * 3, (
|
||||||
"a container was started while .env still named the previous deploy's version"
|
"a container was started while .env still named the previous deploy's version"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user