diff --git a/README.md b/README.md index 19b6dd9..f874d14 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,7 @@ elsewhere. ./deploy.sh 10.0.0.1 # one server ./deploy.sh 10.0.0.1 10.0.0.2 # several, in parallel ./deploy.sh --version=main-1a2b3c4 10.0.0.1 # one-off tag, this run only +./deploy.sh --infra-db 10.0.0.1 # also move the bundled database (see below) ``` Only tags netOrk's CI publishes are accepted: @@ -97,24 +98,52 @@ started earlier pulls whatever image the registry held before, which is stale. hand-typed `docker compose` on the host uses the same images. It does this before any container starts, because the services read `.env` into their environment. A failed pull leaves `.env` as it was. -5. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers, +5. Reconciles the bundled PostgreSQL, if the host has one (`remote/database.sh`, + over ssh's stdin). On the volume it already uses, it is recreated only if its image + or definition changed, so a new minor release arrives with the next deploy. If the + compose files name another volume (a new major version), a normal deploy stops + here before anything changed; see *Moving the bundled database*. +6. Reconciles the broker, `redis` (Valkey): recreated only if its definition changed, + and waited for until healthy, before the engine comes up against it. If it does + not get healthy within 120 s, the deploy stops with the old engine running. +7. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers, `netork-ui`. If the recreate fails, it is tried once more after 5 s (`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it just renamed and leave the rest stopped. If the second attempt fails too, the container states are printed and the deploy fails. -6. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its - definition changed. It never touches `postgres` or `redis`. +8. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its + definition changed. It then removes services netOrk no longer ships, container and volume, where a host still has them: today the bundled `registry` (NetOrk/netork#763). It reports only what it actually removed; on a host that was cleaned earlier it says nothing. -7. Verifies that the containers run exactly the image that was pulled. If they don't, the +9. Verifies that the containers run exactly the image that was pulled. If they don't, the deploy fails. -8. Runs `alembic upgrade head` inside `netork-api`. -9. Prunes unused images. +10. Runs `alembic upgrade head` inside `netork-api`. +11. Prunes unused images. A failing step stops that server's deploy with a non-zero exit, and the other servers carry on. The script exits non-zero if any server failed and names those servers. +## Moving the bundled database + +A new PostgreSQL major version cannot read the old data directory, so netOrk names a +new volume for it, for example `postgres17_data` (NetOrk/netork#1308). Moving onto it +takes `--infra-db`, for one host at an announced time, because the engine is down +while the data moves. + +1. Stops the engine. +2. Counts every table and dumps the database, using `pg_dump -Fc`. The dump stays + in `~/netork/backups/`. +3. Brings the new container up on the new volume, then restores and runs + `ANALYZE`. +4. Compares the counts and the Alembic revision. If either differs, the move fails + and says how to go back. +5. Leaves the old volume as it was. + +**The way back:** deploy the previous netOrk version with `--infra-db`. Its compose +files name the old volume again, which still exists, so the container switches onto +it without a restore. The data is then as it was before the move. + ## Operating notes - **Never delete these Docker volumes:** diff --git a/deploy.sh b/deploy.sh index cad1183..438afed 100644 --- a/deploy.sh +++ b/deploy.sh @@ -2,12 +2,17 @@ # Deploy netOrk to one or more servers in parallel. # # Usage: -# ./deploy.sh [--version=] [server1 server2 ...] +# ./deploy.sh [--version=] [--infra-db] [server1 server2 ...] # # --version overrides the per-server NETORK_VERSION from deploy.env for this # run only — useful to test one commit (--version=main-) without moving # a server to a different release channel. # +# --infra-db lets the bundled PostgreSQL move to the volume the compose files +# name, by dump and restore (a new major version, NetOrk/netork#1308). Without +# it a deploy that would need that move stops before anything changed. Run it +# for one host at an announced time: the engine is down while the data moves. +# # Default targets and registry credentials are read from deploy.env next to # this script (see deploy.env.example). DEPLOY_ENV_FILE points elsewhere. # @@ -178,6 +183,7 @@ release_host_lock() { } VERSION_OVERRIDE="" +DATABASE_MODE=reconcile EXPLICIT_SERVERS=() i=1 while [[ $i -le $# ]]; do @@ -188,6 +194,7 @@ while [[ $i -le $# ]]; do VERSION_OVERRIDE="${!i:-}" ;; --version=*) VERSION_OVERRIDE="${arg#--version=}" ;; + --infra-db) DATABASE_MODE=move ;; -*) echo "Unknown option: ${arg}" >&2 exit 2 @@ -353,6 +360,20 @@ for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; d done REMOTE_ENV + # The bundled database, before broker and engine (remote/database.sh). A + # changed image on the same volume (a minor release) is recreated; a new + # volume (a major release) moves only with --infra-db, and without it the + # deploy stops here, nothing changed and the old engine running. The script + # goes over stdin, so the host needs nothing but bash and docker. + echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." + local db_status=0 + ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ + < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? + if [[ $db_status -ne 0 ]]; then + echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2 + return 1 + fi + # The broker, before the engine that cannot work without it. Plain `up -d` # recreates it only when its compose definition changed, so an unchanged # broker keeps running and keeps its queue. It used to be in no list at all, diff --git a/remote/database.sh b/remote/database.sh new file mode 100644 index 0000000..dbbfdd7 --- /dev/null +++ b/remote/database.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# database.sh : netOrk's bundled PostgreSQL +# on a deploy (NetOrk/netork#1308). deploy.sh runs it on the host, in ~/netork, +# after the compose files and .env are in place and before broker and engine. +# +# - No bundled database (external PostgreSQL): nothing to do. +# - Compose names the volume the container uses: plain `up -d --wait`, which +# recreates only on a changed image or definition (a new minor release). +# - Compose names another volume (a new major version): +# - reconcile: stop here, exit 3, nothing changed; that needs --infra-db. +# - move, volume absent: stop the engine, count and dump the old database +# (the dump stays in ~/netork/backups), bring the new container up on the new +# volume, restore, ANALYZE, compare counts and Alembic revision. +# - move, volume present: the way back; stop the engine and switch onto it. +# The old volume always stays. +# +# Env: REGISTRY_HOST, NETORK_VERSION (for the compose files' substitutions). +set -euo pipefail + +mode=$1 +shift +engine=("$@") +data=/var/lib/postgresql/data +cd ~/netork + +compose() { + docker compose --profile local-db -f docker-compose.yml -f docker-compose.registry.yml "$@" +} +db_container() { + docker ps -aq --filter label=com.docker.compose.project=netork \ + --filter label=com.docker.compose.service=postgres | head -1 +} +up() { compose up -d --no-deps --wait --wait-timeout 300 postgres; } + +cid=$(db_container) +if [ -z "$cid" ]; then + echo "no bundled database (external PostgreSQL): nothing to do" + exit 0 +fi + +current=$(docker inspect "$cid" \ + --format "{{range .Mounts}}{{if eq .Destination \"$data\"}}{{.Name}}{{end}}{{end}}") +config=$(compose config --format json | tr -d ' \n\t') +source=$(grep -o "\"source\":\"[^\"]*\",\"target\":\"$data\"" <<<"$config" \ + | sed 's/^"source":"\([^"]*\)".*/\1/') +target=$(grep -o "\"$source\":{\"name\":\"[^\"]*\"" <<<"$config" | sed 's/.*"name":"\([^"]*\)"$/\1/' || true) +target=${target:-netork_$source} + +if [ "$current" = "$target" ]; then + echo "database stays on $current" + up + exit 0 +fi + +if [ "$mode" != move ]; then + echo "The database would move from volume $current to $target: a new PostgreSQL major" >&2 + echo "version, which only happens by dump and restore. Run deploy.sh --infra-db for" >&2 + echo "this host at an announced time. Nothing was changed." >&2 + exit 3 +fi + +if docker volume inspect "$target" >/dev/null 2>&1; then + echo "switching the database from $current onto the existing volume $target;" + echo "its data is as it was when that volume was last in use" + compose stop "${engine[@]}" + up + echo "database now on $target; $current stays" + exit 0 +fi + +user=$(docker exec "$cid" printenv POSTGRES_USER || echo postgres) +db=$(docker exec "$cid" printenv POSTGRES_DB || echo "$user") +psql_in() { local c=$1; shift; docker exec "$c" psql -U "$user" -d "$db" -XAtq -v ON_ERROR_STOP=1 -c "$@"; } +counts() { # counts : "schema.table=rows" for every table, sorted + local stmt + stmt=$(psql_in "$1" "SELECT string_agg(format('SELECT %L AS t, count(*) AS n FROM %I.%I', schemaname||'.'||relname, schemaname, relname), ' UNION ALL ') FROM pg_stat_user_tables") + psql_in "$1" "SELECT t||'='||n FROM ($stmt) x ORDER BY t" +} + +docker start "$cid" >/dev/null +for _ in $(seq 1 60); do docker exec "$cid" pg_isready -U "$user" -q && break; sleep 1; done + +echo "stopping the engine: ${engine[*]}" +compose stop "${engine[@]}" + +mkdir -p backups +stamp=$(date -u +%Y%m%dT%H%M%SZ) +dump="backups/db-$current-$stamp.dump" +counts "$cid" > "backups/db-$stamp.before" +revision=$(psql_in "$cid" "SELECT version_num FROM alembic_version") +echo "dumping $db from $current to ~/netork/$dump" +docker exec "$cid" pg_dump -U "$user" -Fc "$db" > "$dump" + +echo "bringing the database up on $target" +up +new=$(db_container) +echo "restoring into $target" +docker exec -i "$new" pg_restore -U "$user" -d "$db" --exit-on-error --no-owner --role="$user" < "$dump" +psql_in "$new" ANALYZE +counts "$new" > "backups/db-$stamp.after" + +back="To go back: deploy the previous netOrk version with deploy.sh --infra-db; it switches onto $current, which holds the data as it was before the move." +if ! cmp -s "backups/db-$stamp.before" "backups/db-$stamp.after"; then + echo "The row counts differ after the restore (backups/db-$stamp.before, .after):" >&2 + diff "backups/db-$stamp.before" "backups/db-$stamp.after" | head -20 >&2 || true + echo "$back" >&2 + exit 1 +fi +if [ "$(psql_in "$new" "SELECT version_num FROM alembic_version")" != "$revision" ]; then + echo "The Alembic revision differs after the restore." >&2 + echo "$back" >&2 + exit 1 +fi +echo "database moved from $current to $target: $(wc -l < "backups/db-$stamp.after") tables, counts equal, revision $revision" +echo "the dump stays at ~/netork/$dump; $current stays as the way back" diff --git a/tests/test_deploy_infra_db.py b/tests/test_deploy_infra_db.py new file mode 100644 index 0000000..69ce848 --- /dev/null +++ b/tests/test_deploy_infra_db.py @@ -0,0 +1,84 @@ +"""deploy.sh runs remote/database.sh on every host (NetOrk/netork#1308). + +A normal deploy reconciles the bundled database (`reconcile`); only --infra-db +lets it move to a new volume (`move`). It runs after .env is written and before +the broker and the engine, and a refusal (exit 3: compose wants a new volume) +stops the deploy there, with the old engine still running. + +The remote side is a fake `ssh` that records every command and what it was fed. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" +DATABASE = Path(__file__).resolve().parent.parent / "remote" / "database.sh" + +_FAKE_SSH = """#!/bin/sh +for arg in "$@"; do cmd="$arg"; done +case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac +printf '%s\\n----\\n' "$cmd" >> "{state}/commands" +case "$cmd" in + *"bash -s -- "*) cat > "{state}/database-stdin"; exit {db_exit} ;; +esac +cat > /dev/null +exit 0 +""" + + +def _deploy(tmp_path: Path, *args: str, db_exit: int = 0): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake = bin_dir / "ssh" + fake.write_text(_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{db_exit}", str(db_exit))) + fake.chmod(0o755) + result = subprocess.run( + ["bash", str(DEPLOY), *args, "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + commands = (tmp_path / "commands").read_text().split("\n----\n") + return result, commands + + +def _index(commands: list[str], needle: str) -> int: + return next(i for i, c in enumerate(commands) if needle in c) + + +def test_a_normal_deploy_reconciles_the_database_first(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path) + + assert result.returncode == 0, result.stderr + db = _index(commands, "bash -s -- reconcile netork-api") + assert "NETORK_VERSION='latest-dev'" in commands[db] + assert db < _index(commands, "--wait-timeout 120 redis") < _index(commands, "--force-recreate") + assert (tmp_path / "database-stdin").read_text() == DATABASE.read_text() + + +def test_infra_db_lets_it_move(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path, "--infra-db") + + assert result.returncode == 0, result.stderr + assert any("bash -s -- move netork-api" in c for c in commands) + assert not any("bash -s -- reconcile" in c for c in commands) + + +def test_a_refusal_stops_the_deploy_before_the_engine(tmp_path: Path) -> None: + result, commands = _deploy(tmp_path, db_exit=3) + + assert result.returncode != 0 + assert not any("--force-recreate" in c for c in commands) + assert not any("redis" in c and "up -d" in c for c in commands) diff --git a/tests/test_remote_database.py b/tests/test_remote_database.py new file mode 100644 index 0000000..442d15a --- /dev/null +++ b/tests/test_remote_database.py @@ -0,0 +1,204 @@ +"""The bundled PostgreSQL on a deploy (NetOrk/netork#1308): remote/database.sh. + +deploy.sh runs it on the host before the broker and the engine: + +- no bundled database (external PostgreSQL): nothing to do; +- compose names the volume the container already uses: a plain `up -d --wait`, + which recreates the container only when its image or definition changed, so + a new PostgreSQL minor release reaches the host with the next deploy; +- compose names another volume (a new major version): a normal deploy stops + there, before anything changed, and points at --infra-db; +- with --infra-db and a volume that does not exist yet: the engine is stopped, + the old database is counted and dumped (the dump stays in ~/netork/backups), + the new container comes up on the new volume, the dump is restored, ANALYZE, + and the counts and the Alembic revision must match. The old volume stays; +- with --infra-db and a volume that exists (the way back): the engine is + stopped and the container switches onto it, without a restore. + +The remote side is a fake `docker` that plays the host and records each call. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "database.sh" +ENGINE = ["netork-api", "netork-worker", "netork-beat"] + +_FAKE_DOCKER = r"""#!/bin/bash +state=$FAKE_STATE +printf '%s\n' "$*" >> "$state/calls" +upped() { [ -f "$state/upped" ]; } +case "$1" in + ps) + [ -n "$FAKE_PG_CID" ] || exit 0 + if upped; then echo newcid; else echo "$FAKE_PG_CID"; fi ;; + inspect) echo "$FAKE_CURRENT_VOL" ;; + volume) [ "$FAKE_TARGET_EXISTS" = 1 ] && exit 0 || exit 1 ;; + start) exit 0 ;; + compose) + case "$*" in + *" config "*) printf '%s' "$FAKE_CONFIG" ;; + *" up -d "*) touch "$state/upped" ;; + esac ;; + exec) + shift + [ "$1" = -i ] && shift + shift # container + case "$1" in + printenv) echo netork ;; + pg_isready) exit 0 ;; + pg_dump) echo DUMPDATA ;; + pg_restore) cat > "$state/restored"; exit "${FAKE_RESTORE_EXIT:-0}" ;; + psql) + q="${*: -1}" + case "$q" in + *alembic_version*) if upped; then echo "${FAKE_REV_AFTER:-abc}"; else echo abc; fi ;; + *string_agg*) echo "SELECT 'public.t' AS t, 1 AS n" ;; + ANALYZE) ;; + *"FROM (SELECT 'public.t'"*) if upped; then echo "${FAKE_COUNTS_AFTER:-public.t=5}"; else echo "public.t=5"; fi ;; + *pg_database_size*) echo 1000 ;; + esac ;; + esac ;; +esac +exit 0 +""" + + +def _config(source: str, name: str) -> str: + return json.dumps( + { + "services": { + "postgres": { + "volumes": [ + {"type": "volume", "source": source, "target": "/var/lib/postgresql/data"} + ] + } + }, + "volumes": {source: {"name": name}}, + }, + indent=2, + ) + + +def _run(tmp_path: Path, mode: str = "reconcile", **fake: str): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(_FAKE_DOCKER) + docker.chmod(0o755) + (tmp_path / "netork").mkdir() + env = { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "FAKE_STATE": str(tmp_path), + "FAKE_PG_CID": "oldcid", + "FAKE_CURRENT_VOL": "netork_postgres_data", + "FAKE_CONFIG": _config("postgres17_data", "netork_postgres17_data"), + "FAKE_TARGET_EXISTS": "0", + "REGISTRY_HOST": "registry.example", + "NETORK_VERSION": "latest-dev", + **fake, + } + result = subprocess.run( + ["bash", str(SCRIPT), mode, *ENGINE], + env=env, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + calls_file = tmp_path / "calls" + calls = calls_file.read_text().splitlines() if calls_file.exists() else [] + return result, calls + + +def _first(calls: list[str], needle: str) -> int: + return next(i for i, c in enumerate(calls) if needle in c) + + +def test_an_external_database_is_left_alone(tmp_path: Path) -> None: + result, calls = _run(tmp_path, FAKE_PG_CID="") + + assert result.returncode == 0, result.stderr + assert not any(c.startswith("compose") for c in calls) + assert "external" in result.stdout + + +def test_the_same_volume_gets_a_plain_reconcile(tmp_path: Path) -> None: + result, calls = _run(tmp_path, FAKE_CURRENT_VOL="netork_postgres17_data") + + assert result.returncode == 0, result.stderr + assert any("up -d --no-deps --wait" in c and c.endswith(" postgres") for c in calls) + assert not any(" stop " in c for c in calls) + assert not any("pg_dump" in c for c in calls) + + +def test_a_new_volume_stops_a_normal_deploy_before_anything_changed(tmp_path: Path) -> None: + result, calls = _run(tmp_path) + + assert result.returncode == 3 + assert "--infra-db" in result.stderr + assert not any(" up " in c or " stop " in c or "pg_dump" in c for c in calls) + + +def test_infra_db_moves_the_data_by_dump_and_restore(tmp_path: Path) -> None: + result, calls = _run(tmp_path, mode="move") + + assert result.returncode == 0, result.stderr + stop = _first(calls, " stop netork-api netork-worker netork-beat") + dump = _first(calls, "pg_dump") + up = _first(calls, " up -d --no-deps --wait") + restore = _first(calls, "pg_restore") + assert stop < dump < up < restore + assert any(c.endswith("ANALYZE") for c in calls) + assert (tmp_path / "restored").read_text() == "DUMPDATA\n" + dumps = list((tmp_path / "netork" / "backups").glob("db-netork_postgres_data-*.dump")) + assert len(dumps) == 1, "the dump must stay on the host" + assert "netork_postgres_data" in result.stdout # the old volume is named as the way back + + +def test_counts_that_differ_fail_the_move_and_say_how_back(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_COUNTS_AFTER="public.t=4") + + assert result.returncode != 0 + assert "--infra-db" in result.stderr and "netork_postgres_data" in result.stderr + + +def test_another_revision_fails_the_move(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_REV_AFTER="other") + + assert result.returncode != 0 + + +def test_a_failed_restore_fails_the_move(tmp_path: Path) -> None: + result, _ = _run(tmp_path, mode="move", FAKE_RESTORE_EXIT="1") + + assert result.returncode != 0 + + +def test_an_existing_volume_is_switched_onto_without_a_restore(tmp_path: Path) -> None: + """The way back: compose names the old volume again, and it is still there.""" + result, calls = _run( + tmp_path, + mode="move", + FAKE_CURRENT_VOL="netork_postgres17_data", + FAKE_CONFIG=_config("postgres_data", "netork_postgres_data"), + FAKE_TARGET_EXISTS="1", + ) + + assert result.returncode == 0, result.stderr + assert _first(calls, " stop netork-api") < _first(calls, " up -d --no-deps --wait") + assert not any("pg_dump" in c or "pg_restore" in c for c in calls) + + +def test_the_counts_compared_are_real(tmp_path: Path) -> None: + """Guards the fake: equal counts must come from rows, not from two empty lists.""" + result, _ = _run(tmp_path, mode="move") + + assert result.returncode == 0, result.stderr + before = next((tmp_path / "netork" / "backups").glob("db-*.before")).read_text() + assert before == "public.t=5\n"