From 6e3ecb8680401468b72e796477d0ca01767d5a8d Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Tue, 20 Oct 2026 23:15:48 +0200 Subject: [PATCH 1/3] feat: the bundled database on every deploy, and --infra-db to move it (NetOrk/netork#1308) remote/database.sh runs on the host (over ssh's stdin) after .env and before broker and engine: - no bundled database: nothing to do; - the volume it already uses: plain `up -d --wait`, recreated only on a changed image or definition, so a new PostgreSQL minor release arrives with the next deploy; - another volume (a new major version): a normal deploy stops there, nothing changed; with --infra-db the engine stops, the old database is counted and dumped (kept in ~/netork/backups), the new container comes up on the new volume, restore, ANALYZE, counts and Alembic revision must match. The old volume stays, and deploying the previous version with --infra-db switches back onto it without a restore. Checked on .50 in reconcile mode: the running postgres:16 container stayed as it was. README describes the steps and the way back. --- README.md | 41 ++++++- deploy.sh | 23 +++- remote/database.sh | 115 +++++++++++++++++++ tests/test_deploy_infra_db.py | 84 ++++++++++++++ tests/test_remote_database.py | 204 ++++++++++++++++++++++++++++++++++ 5 files changed, 460 insertions(+), 7 deletions(-) create mode 100644 remote/database.sh create mode 100644 tests/test_deploy_infra_db.py create mode 100644 tests/test_remote_database.py diff --git a/README.md b/README.md index 19b6dd9..f874d14 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ elsewhere. ./deploy.sh 10.0.0.1 # one server ./deploy.sh 10.0.0.1 10.0.0.2 # several, in parallel ./deploy.sh --version=main-1a2b3c4 10.0.0.1 # one-off tag, this run only +./deploy.sh --infra-db 10.0.0.1 # also move the bundled database (see below) ``` Only tags netOrk's CI publishes are accepted: @@ -97,24 +98,52 @@ started earlier pulls whatever image the registry held before, which is stale. hand-typed `docker compose` on the host uses the same images. It does this before any container starts, because the services read `.env` into their environment. A failed pull leaves `.env` as it was. -5. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers, +5. Reconciles the bundled PostgreSQL, if the host has one (`remote/database.sh`, + over ssh's stdin). On the volume it already uses, it is recreated only if its image + or definition changed, so a new minor release arrives with the next deploy. If the + compose files name another volume (a new major version), a normal deploy stops + here before anything changed; see *Moving the bundled database*. +6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed, + and waited for until healthy, before the engine comes up against it. If it does + not get healthy within 120 s, the deploy stops with the old engine running. +7. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers, `netork-ui`. If the recreate fails, it is tried once more after 5 s (`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it just renamed and leave the rest stopped. If the second attempt fails too, the container states are printed and the deploy fails. -6. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its - definition changed. It never touches `postgres` or `redis`. +8. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its + definition changed. It then removes services netOrk no longer ships, container and volume, where a host still has them: today the bundled `registry` (NetOrk/netork#763). It reports only what it actually removed; on a host that was cleaned earlier it says nothing. -7. Verifies that the containers run exactly the image that was pulled. If they don't, the +9. Verifies that the containers run exactly the image that was pulled. If they don't, the deploy fails. -8. Runs `alembic upgrade head` inside `netork-api`. -9. Prunes unused images. +10. Runs `alembic upgrade head` inside `netork-api`. +11. Prunes unused images. A failing step stops that server's deploy with a non-zero exit, and the other servers carry on. The script exits non-zero if any server failed and names those servers. +## Moving the bundled database + +A new PostgreSQL major version cannot read the old data directory, so netOrk names a +new volume for it, for example `postgres17_data` (NetOrk/netork#1308). Moving onto it +takes `--infra-db`, for one host at an announced time, because the engine is down +while the data moves. + +1. Stops the engine. +2. Counts every table and dumps the database, using `pg_dump -Fc`. The dump stays + in `~/netork/backups/`. +3. Brings the new container up on the new volume, then restores and runs + `ANALYZE`. +4. Compares the counts and the Alembic revision. If either differs, the move fails + and says how to go back. +5. Leaves the old volume as it was. + +**The way back:** deploy the previous netOrk version with `--infra-db`. Its compose +files name the old volume again, which still exists, so the container switches onto +it without a restore. The data is then as it was before the move. + ## Operating notes - **Never delete these Docker volumes:** diff --git a/deploy.sh b/deploy.sh index cad1183..438afed 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,12 +2,17 @@ # Deploy netOrk to one or more servers in parallel. # # Usage: -# ./deploy.sh [--version=] [server1 server2 ...] +# ./deploy.sh [--version=] [--infra-db] [server1 server2 ...] # # --version overrides the per-server NETORK_VERSION from deploy.env for this # run only — useful to test one commit (--version=main-) without moving # a server to a different release channel. # +# --infra-db lets the bundled PostgreSQL move to the volume the compose files +# name, by dump and restore (a new major version, NetOrk/netork#1308). Without +# it a deploy that would need that move stops before anything changed. Run it +# for one host at an announced time: the engine is down while the data moves. +# # Default targets and registry credentials are read from deploy.env next to # this script (see deploy.env.example). DEPLOY_ENV_FILE points elsewhere. # @@ -178,6 +183,7 @@ release_host_lock() { } VERSION_OVERRIDE="" +DATABASE_MODE=reconcile EXPLICIT_SERVERS=() i=1 while [[ $i -le $# ]]; do @@ -188,6 +194,7 @@ while [[ $i -le $# ]]; do VERSION_OVERRIDE="${!i:-}" ;; --version=*) VERSION_OVERRIDE="${arg#--version=}" ;; + --infra-db) DATABASE_MODE=move ;; -*) echo "Unknown option: ${arg}" >&2 exit 2 @@ -353,6 +360,20 @@ for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; d done REMOTE_ENV + # The bundled database, before broker and engine (remote/database.sh). A + # changed image on the same volume (a minor release) is recreated; a new + # volume (a major release) moves only with --infra-db, and without it the + # deploy stops here, nothing changed and the old engine running. The script + # goes over stdin, so the host needs nothing but bash and docker. + echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." + local db_status=0 + ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ + < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? + if [[ $db_status -ne 0 ]]; then + echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2 + return 1 + fi + # The broker, before the engine that cannot work without it. Plain `up -d` # recreates it only when its compose definition changed, so an unchanged # broker keeps running and keeps its queue. It used to be in no list at all, diff --git a/remote/database.sh b/remote/database.sh new file mode 100644 index 0000000..dbbfdd7 --- /dev/null +++ b/remote/database.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# database.sh : netOrk's bundled PostgreSQL +# on a deploy (NetOrk/netork#1308). deploy.sh runs it on the host, in ~/netork, +# after the compose files and .env are in place and before broker and engine. +# +# - No bundled database (external PostgreSQL): nothing to do. +# - Compose names the volume the container uses: plain `up -d --wait`, which +# recreates only on a changed image or definition (a new minor release). +# - Compose names another volume (a new major version): +# - reconcile: stop here, exit 3, nothing changed; that needs --infra-db. +# - move, volume absent: stop the engine, count and dump the old database +# (the dump stays in ~/netork/backups), bring the new container up on the new +# volume, restore, ANALYZE, compare counts and Alembic revision. +# - move, volume present: the way back; stop the engine and switch onto it. +# The old volume always stays. +# +# Env: REGISTRY_HOST, NETORK_VERSION (for the compose files' substitutions). +set -euo pipefail + +mode=$1 +shift +engine=("$@") +data=/var/lib/postgresql/data +cd ~/netork + +compose() { + docker compose --profile local-db -f docker-compose.yml -f docker-compose.registry.yml "$@" +} +db_container() { + docker ps -aq --filter label=com.docker.compose.project=netork \ + --filter label=com.docker.compose.service=postgres | head -1 +} +up() { compose up -d --no-deps --wait --wait-timeout 300 postgres; } + +cid=$(db_container) +if [ -z "$cid" ]; then + echo "no bundled database (external PostgreSQL): nothing to do" + exit 0 +fi + +current=$(docker inspect "$cid" \ + --format "{{range .Mounts}}{{if eq .Destination \"$data\"}}{{.Name}}{{end}}{{end}}") +config=$(compose config --format json | tr -d ' \n\t') +source=$(grep -o "\"source\":\"[^\"]*\",\"target\":\"$data\"" <<<"$config" \ + | sed 's/^"source":"\([^"]*\)".*/\1/') +target=$(grep -o "\"$source\":{\"name\":\"[^\"]*\"" <<<"$config" | sed 's/.*"name":"\([^"]*\)"$/\1/' || true) +target=${target:-netork_$source} + +if [ "$current" = "$target" ]; then + echo "database stays on $current" + up + exit 0 +fi + +if [ "$mode" != move ]; then + echo "The database would move from volume $current to $target: a new PostgreSQL major" >&2 + echo "version, which only happens by dump and restore. Run deploy.sh --infra-db for" >&2 + echo "this host at an announced time. Nothing was changed." >&2 + exit 3 +fi + +if docker volume inspect "$target" >/dev/null 2>&1; then + echo "switching the database from $current onto the existing volume $target;" + echo "its data is as it was when that volume was last in use" + compose stop "${engine[@]}" + up + echo "database now on $target; $current stays" + exit 0 +fi + +user=$(docker exec "$cid" printenv POSTGRES_USER || echo postgres) +db=$(docker exec "$cid" printenv POSTGRES_DB || echo "$user") +psql_in() { local c=$1; shift; docker exec "$c" psql -U "$user" -d "$db" -XAtq -v ON_ERROR_STOP=1 -c "$@"; } +counts() { # counts : "schema.table=rows" for every table, sorted + local stmt + stmt=$(psql_in "$1" "SELECT string_agg(format('SELECT %L AS t, count(*) AS n FROM %I.%I', schemaname||'.'||relname, schemaname, relname), ' UNION ALL ') FROM pg_stat_user_tables") + psql_in "$1" "SELECT t||'='||n FROM ($stmt) x ORDER BY t" +} + +docker start "$cid" >/dev/null +for _ in $(seq 1 60); do docker exec "$cid" pg_isready -U "$user" -q && break; sleep 1; done + +echo "stopping the engine: ${engine[*]}" +compose stop "${engine[@]}" + +mkdir -p backups +stamp=$(date -u +%Y%m%dT%H%M%SZ) +dump="backups/db-$current-$stamp.dump" +counts "$cid" > "backups/db-$stamp.before" +revision=$(psql_in "$cid" "SELECT version_num FROM alembic_version") +echo "dumping $db from $current to ~/netork/$dump" +docker exec "$cid" pg_dump -U "$user" -Fc "$db" > "$dump" + +echo "bringing the database up on $target" +up +new=$(db_container) +echo "restoring into $target" +docker exec -i "$new" pg_restore -U "$user" -d "$db" --exit-on-error --no-owner --role="$user" < "$dump" +psql_in "$new" ANALYZE +counts "$new" > "backups/db-$stamp.after" + +back="To go back: deploy the previous netOrk version with deploy.sh --infra-db; it switches onto $current, which holds the data as it was before the move." +if ! cmp -s "backups/db-$stamp.before" "backups/db-$stamp.after"; then + echo "The row counts differ after the restore (backups/db-$stamp.before, .after):" >&2 + diff "backups/db-$stamp.before" "backups/db-$stamp.after" | head -20 >&2 || true + echo "$back" >&2 + exit 1 +fi +if [ "$(psql_in "$new" "SELECT version_num FROM alembic_version")" != "$revision" ]; then + echo "The Alembic revision differs after the restore." >&2 + echo "$back" >&2 + exit 1 +fi +echo "database moved from $current to $target: $(wc -l < "backups/db-$stamp.after") tables, counts equal, revision $revision" +echo "the dump stays at ~/netork/$dump; $current stays as the way back" diff --git a/tests/test_deploy_infra_db.py b/tests/test_deploy_infra_db.py new file mode 100644 index 0000000..69ce848 --- /dev/null +++ b/tests/test_deploy_infra_db.py @@ -0,0 +1,84 @@ +"""deploy.sh runs remote/database.sh on every host (NetOrk/netork#1308). + +A normal deploy reconciles the bundled database (`reconcile`); only --infra-db +lets it move to a new volume (`move`). It runs after .env is written and before +the broker and the engine, and a refusal (exit 3: compose wants a new volume) +stops the deploy there, with the old engine still running. + +The remote side is a fake `ssh` that records every command and what it was fed. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" +DATABASE = Path(__file__).resolve().parent.parent / "remote" / "database.sh" + +_FAKE_SSH = """#!/bin/sh +for arg in "$@"; do cmd="$arg"; done +case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac +printf '%s\\n----\\n' "$cmd" >> "{state}/commands" +case "$cmd" in + *"bash -s -- "*) cat > "{state}/database-stdin"; exit {db_exit} ;; +esac +cat > /dev/null +exit 0 +""" + + +def _deploy(tmp_path: Path, *args: str, db_exit: int = 0): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake = bin_dir / "ssh" + fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{db_exit}", str(db_exit))) + fake.chmod(0o755) + result = subprocess.run( + ["bash", str(DEPLOY), *args, "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + commands = (tmp_path / "commands").read_text().split("\n----\n") + return result, commands + + +def _index(commands: list[str], needle: str) -> int: + return next(i for i, c in enumerate(commands) if needle in c) + + +def test_a_normal_deploy_reconciles_the_database_first(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path) + + assert result.returncode == 0, result.stderr + db = _index(commands, "bash -s -- reconcile netork-api") + assert "NETORK_VERSION='latest-dev'" in commands[db] + assert db < _index(commands, "--wait-timeout 120 redis") < _index(commands, "--force-recreate") + assert (tmp_path / "database-stdin").read_text() == DATABASE.read_text() + + +def test_infra_db_lets_it_move(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path, "--infra-db") + + assert result.returncode == 0, result.stderr + assert any("bash -s -- move netork-api" in c for c in commands) + assert not any("bash -s -- reconcile" in c for c in commands) + + +def test_a_refusal_stops_the_deploy_before_the_engine(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path, db_exit=3) + + assert result.returncode != 0 + assert not any("--force-recreate" in c for c in commands) + assert not any("redis" in c and "up -d" in c for c in commands) diff --git a/tests/test_remote_database.py b/tests/test_remote_database.py new file mode 100644 index 0000000..442d15a --- /dev/null +++ b/tests/test_remote_database.py @@ -0,0 +1,204 @@ +"""The bundled PostgreSQL on a deploy (NetOrk/netork#1308): remote/database.sh. + +deploy.sh runs it on the host before the broker and the engine: + +- no bundled database (external PostgreSQL): nothing to do; +- compose names the volume the container already uses: a plain `up -d --wait`, + which recreates the container only when its image or definition changed, so + a new PostgreSQL minor release reaches the host with the next deploy; +- compose names another volume (a new major version): a normal deploy stops + there, before anything changed, and points at --infra-db; +- with --infra-db and a volume that does not exist yet: the engine is stopped, + the old database is counted and dumped (the dump stays in ~/netork/backups), + the new container comes up on the new volume, the dump is restored, ANALYZE, + and the counts and the Alembic revision must match. The old volume stays; +- with --infra-db and a volume that exists (the way back): the engine is + stopped and the container switches onto it, without a restore. + +The remote side is a fake `docker` that plays the host and records each call. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "database.sh" +ENGINE = ["netork-api", "netork-worker", "netork-beat"] + +_FAKE_DOCKER = r"""#!/bin/bash +state=$FAKE_STATE +printf '%s\n' "$*" >> "$state/calls" +upped() { [ -f "$state/upped" ]; } +case "$1" in + ps) + [ -n "$FAKE_PG_CID" ] || exit 0 + if upped; then echo newcid; else echo "$FAKE_PG_CID"; fi ;; + inspect) echo "$FAKE_CURRENT_VOL" ;; + volume) [ "$FAKE_TARGET_EXISTS" = 1 ] && exit 0 || exit 1 ;; + start) exit 0 ;; + compose) + case "$*" in + *" config "*) printf '%s' "$FAKE_CONFIG" ;; + *" up -d "*) touch "$state/upped" ;; + esac ;; + exec) + shift + [ "$1" = -i ] && shift + shift # container + case "$1" in + printenv) echo netork ;; + pg_isready) exit 0 ;; + pg_dump) echo DUMPDATA ;; + pg_restore) cat > "$state/restored"; exit "${FAKE_RESTORE_EXIT:-0}" ;; + psql) + q="${*: -1}" + case "$q" in + *alembic_version*) if upped; then echo "${FAKE_REV_AFTER:-abc}"; else echo abc; fi ;; + *string_agg*) echo "SELECT 'public.t' AS t, 1 AS n" ;; + ANALYZE) ;; + *"FROM (SELECT 'public.t'"*) if upped; then echo "${FAKE_COUNTS_AFTER:-public.t=5}"; else echo "public.t=5"; fi ;; + *pg_database_size*) echo 1000 ;; + esac ;; + esac ;; +esac +exit 0 +""" + + +def _config(source: str, name: str) -> str: + return json.dumps( + { + "services": { + "postgres": { + "volumes": [ + {"type": "volume", "source": source, "target": "/var/lib/postgresql/data"} + ] + } + }, + "volumes": {source: {"name": name}}, + }, + indent=2, + ) + + +def _run(tmp_path: Path, mode: str = "reconcile", **fake: str): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(_FAKE_DOCKER) + docker.chmod(0o755) + (tmp_path / "netork").mkdir() + env = { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "FAKE_STATE": str(tmp_path), + "FAKE_PG_CID": "oldcid", + "FAKE_CURRENT_VOL": "netork_postgres_data", + "FAKE_CONFIG": _config("postgres17_data", "netork_postgres17_data"), + "FAKE_TARGET_EXISTS": "0", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + **fake, + } + result = subprocess.run( + ["bash", str(SCRIPT), mode, *ENGINE], + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + calls_file = tmp_path / "calls" + calls = calls_file.read_text().splitlines() if calls_file.exists() else [] + return result, calls + + +def _first(calls: list[str], needle: str) -> int: + return next(i for i, c in enumerate(calls) if needle in c) + + +def test_an_external_database_is_left_alone(tmp_path: Path) -> None: + result, calls = _run(tmp_path, FAKE_PG_CID="") + + assert result.returncode == 0, result.stderr + assert not any(c.startswith("compose") for c in calls) + assert "external" in result.stdout + + +def test_the_same_volume_gets_a_plain_reconcile(tmp_path: Path) -> None: + result, calls = _run(tmp_path, FAKE_CURRENT_VOL="netork_postgres17_data") + + assert result.returncode == 0, result.stderr + assert any("up -d --no-deps --wait" in c and c.endswith(" postgres") for c in calls) + assert not any(" stop " in c for c in calls) + assert not any("pg_dump" in c for c in calls) + + +def test_a_new_volume_stops_a_normal_deploy_before_anything_changed(tmp_path: Path) -> None: + result, calls = _run(tmp_path) + + assert result.returncode == 3 + assert "--infra-db" in result.stderr + assert not any(" up " in c or " stop " in c or "pg_dump" in c for c in calls) + + +def test_infra_db_moves_the_data_by_dump_and_restore(tmp_path: Path) -> None: + result, calls = _run(tmp_path, mode="move") + + assert result.returncode == 0, result.stderr + stop = _first(calls, " stop netork-api netork-worker netork-beat") + dump = _first(calls, "pg_dump") + up = _first(calls, " up -d --no-deps --wait") + restore = _first(calls, "pg_restore") + assert stop < dump < up < restore + assert any(c.endswith("ANALYZE") for c in calls) + assert (tmp_path / "restored").read_text() == "DUMPDATA\n" + dumps = list((tmp_path / "netork" / "backups").glob("db-netork_postgres_data-*.dump")) + assert len(dumps) == 1, "the dump must stay on the host" + assert "netork_postgres_data" in result.stdout # the old volume is named as the way back + + +def test_counts_that_differ_fail_the_move_and_say_how_back(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_COUNTS_AFTER="public.t=4") + + assert result.returncode != 0 + assert "--infra-db" in result.stderr and "netork_postgres_data" in result.stderr + + +def test_another_revision_fails_the_move(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_REV_AFTER="other") + + assert result.returncode != 0 + + +def test_a_failed_restore_fails_the_move(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_RESTORE_EXIT="1") + + assert result.returncode != 0 + + +def test_an_existing_volume_is_switched_onto_without_a_restore(tmp_path: Path) -> None: + """The way back: compose names the old volume again, and it is still there.""" + result, calls = _run( + tmp_path, + mode="move", + FAKE_CURRENT_VOL="netork_postgres17_data", + FAKE_CONFIG=_config("postgres_data", "netork_postgres_data"), + FAKE_TARGET_EXISTS="1", + ) + + assert result.returncode == 0, result.stderr + assert _first(calls, " stop netork-api") < _first(calls, " up -d --no-deps --wait") + assert not any("pg_dump" in c or "pg_restore" in c for c in calls) + + +def test_the_counts_compared_are_real(tmp_path: Path) -> None: + """Guards the fake: equal counts must come from rows, not from two empty lists.""" + result, _ = _run(tmp_path, mode="move") + + assert result.returncode == 0, result.stderr + before = next((tmp_path / "netork" / "backups").glob("db-*.before")).read_text() + assert before == "public.t=5\n" From 4d3a7eb1f4a6d8570301e1aba4143681d3d0c58b Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Wed, 21 Oct 2026 00:03:45 +0200 Subject: [PATCH 2/3] fix: shellcheck clean; remote/*.sh is checked too SC2029 at the database step is intended: mode, services and version expand on the client side. CI now runs shellcheck on remote/database.sh as well. --- .gitea/workflows/ci.yml | 2 +- deploy.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index d4a6ebb..f0f6f73 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: run: pip install pytest==9.1.1 ruff==0.16.9 shellcheck-py==0.11.0.1 - name: shellcheck - run: shellcheck deploy.sh + run: shellcheck deploy.sh remote/*.sh - name: ruff run: | diff --git a/deploy.sh b/deploy.sh index 438afed..bf8b700 100644 --- a/deploy.sh +++ b/deploy.sh @@ -367,6 +367,7 @@ REMOTE_ENV # goes over stdin, so the host needs nothing but bash and docker. echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." local db_status=0 + # shellcheck disable=SC2029 # mode, services and version expand here on purpose ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? if [[ $db_status -ne 0 ]]; then From 1e87478f7fbe6400e6ad8020f819ca7640ff096a Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Wed, 21 Oct 2026 00:44:15 +0200 Subject: [PATCH 3/3] fix: a refused database move leaves the host as it was On .50 a deploy refused by the database step had already copied the refused version's compose files and written its NETORK_VERSION to .env: a hand-typed `docker compose up -d` would have carried postgres back to the old volume. The database step now runs before .env is written, the previous compose files are kept as .prev while the new ones are fetched, and a refusal (exit 3) puts them back. Checked on .50: checksums of both compose files and .env unchanged after a refused deploy. --- README.md | 4 +++- deploy.sh | 42 ++++++++++++++++++++++------------- tests/test_deploy_infra_db.py | 27 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f874d14..f7b32eb 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,9 @@ started earlier pulls whatever image the registry held before, which is stale. over ssh's stdin). On the volume it already uses, it is recreated only if its image or definition changed, so a new minor release arrives with the next deploy. If the compose files name another volume (a new major version), a normal deploy stops - here before anything changed; see *Moving the bundled database*. + here before anything changed: the host keeps its previous compose files and + `.env`, so a hand-typed `docker compose up -d` cannot move the database either. + See *Moving the bundled database*. This step runs before `.env` is written. 6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed, and waited for until healthy, before the engine comes up against it. If it does not get healthy within 120 s, the deploy stops with the old engine running. diff --git a/deploy.sh b/deploy.sh index bf8b700..ea87237 100644 --- a/deploy.sh +++ b/deploy.sh @@ -301,7 +301,9 @@ deploy_steps() { docker run --rm --entrypoint cat '${ENGINE_IMAGE}' /app/deploy/\$f > \$f.new \ || { rm -f \$f.new; echo \"${ENGINE_IMAGE} carries no /app/deploy/\$f — the tag predates compose files in the engine image\"; exit 3; }; \ done && \ - for f in ${COMPOSE_FILES}; do mv \$f.new \$f; done" + for f in ${COMPOSE_FILES}; do \ + if [ -f \"\$f\" ]; then cp \"\$f\" \"\$f.prev\"; fi; mv \$f.new \$f; \ + done" # Services running netOrk's own images. flower belongs here: it is the engine # image with a different command, so leaving it out let the monitoring UI run @@ -324,6 +326,29 @@ deploy_steps() { REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}" + # The bundled database, before .env, broker and engine (remote/database.sh). A + # changed image on the same volume (a minor release) is recreated; a new + # volume (a major release) moves only with --infra-db, and without it the + # deploy stops here, nothing changed and the old engine running. The script + # goes over stdin, so the host needs nothing but bash and docker. + echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." + local db_status=0 + # shellcheck disable=SC2029 # mode, services and version expand here on purpose + ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ + < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? + if [[ $db_status -eq 3 ]]; then + # Refused, nothing changed: the host keeps the compose files it had, so a + # hand-typed `docker compose up -d` cannot carry the database to the other + # volume. .env is written only after this step. + ssh -n "$SERVER" "cd ~/netork && for f in ${COMPOSE_FILES}; do if [ -f \"\$f.prev\" ]; then mv \"\$f.prev\" \"\$f\"; fi; done" || true + echo "[${SERVER}] FAILED: the database step refused; the host is as it was" >&2 + return 1 + fi + if [[ $db_status -ne 0 ]]; then + echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2 + return 1 + fi + # docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project # .env when they are not in the environment. This script passes both inline, # but a hand-typed `docker compose up -d flower` on the host needs them there @@ -360,21 +385,6 @@ for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; d done REMOTE_ENV - # The bundled database, before broker and engine (remote/database.sh). A - # changed image on the same volume (a minor release) is recreated; a new - # volume (a major release) moves only with --infra-db, and without it the - # deploy stops here, nothing changed and the old engine running. The script - # goes over stdin, so the host needs nothing but bash and docker. - echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." - local db_status=0 - # shellcheck disable=SC2029 # mode, services and version expand here on purpose - ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ - < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? - if [[ $db_status -ne 0 ]]; then - echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2 - return 1 - fi - # The broker, before the engine that cannot work without it. Plain `up -d` # recreates it only when its compose definition changed, so an unchanged # broker keeps running and keeps its queue. It used to be in no list at all, diff --git a/tests/test_deploy_infra_db.py b/tests/test_deploy_infra_db.py index 69ce848..b972264 100644 --- a/tests/test_deploy_infra_db.py +++ b/tests/test_deploy_infra_db.py @@ -82,3 +82,30 @@ def test_a_refusal_stops_the_deploy_before_the_engine(tmp_path: Path) -> None: assert result.returncode != 0 assert not any("--force-recreate" in c for c in commands) assert not any("redis" in c and "up -d" in c for c in commands) + + +def test_the_database_is_checked_before_env_names_the_new_version(tmp_path: Path) -> None: + """A refused deploy must leave .env as it was: the check runs first.""" + _, commands = _deploy(tmp_path) + + env_write = next(i for i, c in enumerate(commands) if c.strip() == "bash -s") + assert _index(commands, "bash -s -- reconcile") < env_write + + +def test_the_previous_compose_files_are_kept_while_the_new_ones_are_fetched(tmp_path: Path) -> None: + _, commands = _deploy(tmp_path) + + fetch = commands[_index(commands, "/app/deploy/")] + assert 'cp "$f" "$f.prev"' in fetch + + +def test_a_refusal_puts_the_previous_compose_files_back(tmp_path: Path) -> None: + """Refused (exit 3) means nothing changed; the host's compose files must not + name the refused version either, or a hand-typed `docker compose up -d` + would carry the database to the other volume (seen on .50, 2026-10-21).""" + result, commands = _deploy(tmp_path, db_exit=3) + + assert result.returncode != 0 + restore = [c for c in commands if 'mv "$f.prev" "$f"' in c] + assert restore, "the previous compose files are not put back" + assert _index(commands, 'mv "$f.prev" "$f"') > _index(commands, "bash -s -- reconcile")