Files
deploy/tests/test_remote_database.py
Christian Manivong 6e3ecb8680
CI / check (pull_request) Failing after 28s
feat: the bundled database on every deploy, and --infra-db to move it (NetOrk/netork#1308)
remote/database.sh runs on the host (over ssh's stdin) after .env and before
broker and engine:

- no bundled database: nothing to do;
- the volume it already uses: plain `up -d --wait`, recreated only on a changed
  image or definition, so a new PostgreSQL minor release arrives with the next
  deploy;
- another volume (a new major version): a normal deploy stops there, nothing
  changed; with --infra-db the engine stops, the old database is counted and
  dumped (kept in ~/netork/backups), the new container comes up on the new
  volume, restore, ANALYZE, counts and Alembic revision must match. The old
  volume stays, and deploying the previous version with --infra-db switches
  back onto it without a restore.

Checked on .50 in reconcile mode: the running postgres:16 container stayed as
it was. README describes the steps and the way back.
2026-10-20 23:15:48 +02:00

205 lines
7.2 KiB
Python

"""The bundled PostgreSQL on a deploy (NetOrk/netork#1308): remote/database.sh.
deploy.sh runs it on the host before the broker and the engine:
- no bundled database (external PostgreSQL): nothing to do;
- compose names the volume the container already uses: a plain `up -d --wait`,
which recreates the container only when its image or definition changed, so
a new PostgreSQL minor release reaches the host with the next deploy;
- compose names another volume (a new major version): a normal deploy stops
there, before anything changed, and points at --infra-db;
- with --infra-db and a volume that does not exist yet: the engine is stopped,
the old database is counted and dumped (the dump stays in ~/netork/backups),
the new container comes up on the new volume, the dump is restored, ANALYZE,
and the counts and the Alembic revision must match. The old volume stays;
- with --infra-db and a volume that exists (the way back): the engine is
stopped and the container switches onto it, without a restore.
The remote side is a fake `docker` that plays the host and records each call.
"""
from __future__ import annotations
import json
import os
import subprocess
from pathlib import Path
SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "database.sh"
ENGINE = ["netork-api", "netork-worker", "netork-beat"]
_FAKE_DOCKER = r"""#!/bin/bash
state=$FAKE_STATE
printf '%s\n' "$*" >> "$state/calls"
upped() { [ -f "$state/upped" ]; }
case "$1" in
ps)
[ -n "$FAKE_PG_CID" ] || exit 0
if upped; then echo newcid; else echo "$FAKE_PG_CID"; fi ;;
inspect) echo "$FAKE_CURRENT_VOL" ;;
volume) [ "$FAKE_TARGET_EXISTS" = 1 ] && exit 0 || exit 1 ;;
start) exit 0 ;;
compose)
case "$*" in
*" config "*) printf '%s' "$FAKE_CONFIG" ;;
*" up -d "*) touch "$state/upped" ;;
esac ;;
exec)
shift
[ "$1" = -i ] && shift
shift # container
case "$1" in
printenv) echo netork ;;
pg_isready) exit 0 ;;
pg_dump) echo DUMPDATA ;;
pg_restore) cat > "$state/restored"; exit "${FAKE_RESTORE_EXIT:-0}" ;;
psql)
q="${*: -1}"
case "$q" in
*alembic_version*) if upped; then echo "${FAKE_REV_AFTER:-abc}"; else echo abc; fi ;;
*string_agg*) echo "SELECT 'public.t' AS t, 1 AS n" ;;
ANALYZE) ;;
*"FROM (SELECT 'public.t'"*) if upped; then echo "${FAKE_COUNTS_AFTER:-public.t=5}"; else echo "public.t=5"; fi ;;
*pg_database_size*) echo 1000 ;;
esac ;;
esac ;;
esac
exit 0
"""
def _config(source: str, name: str) -> str:
return json.dumps(
{
"services": {
"postgres": {
"volumes": [
{"type": "volume", "source": source, "target": "/var/lib/postgresql/data"}
]
}
},
"volumes": {source: {"name": name}},
},
indent=2,
)
def _run(tmp_path: Path, mode: str = "reconcile", **fake: str):
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
docker = bin_dir / "docker"
docker.write_text(_FAKE_DOCKER)
docker.chmod(0o755)
(tmp_path / "netork").mkdir()
env = {
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"FAKE_STATE": str(tmp_path),
"FAKE_PG_CID": "oldcid",
"FAKE_CURRENT_VOL": "netork_postgres_data",
"FAKE_CONFIG": _config("postgres17_data", "netork_postgres17_data"),
"FAKE_TARGET_EXISTS": "0",
"REGISTRY_HOST": "registry.example",
"NETORK_VERSION": "latest-dev",
**fake,
}
result = subprocess.run(
["bash", str(SCRIPT), mode, *ENGINE],
env=env,
capture_output=True,
text=True,
timeout=60,
check=False,
)
calls_file = tmp_path / "calls"
calls = calls_file.read_text().splitlines() if calls_file.exists() else []
return result, calls
def _first(calls: list[str], needle: str) -> int:
return next(i for i, c in enumerate(calls) if needle in c)
def test_an_external_database_is_left_alone(tmp_path: Path) -> None:
result, calls = _run(tmp_path, FAKE_PG_CID="")
assert result.returncode == 0, result.stderr
assert not any(c.startswith("compose") for c in calls)
assert "external" in result.stdout
def test_the_same_volume_gets_a_plain_reconcile(tmp_path: Path) -> None:
result, calls = _run(tmp_path, FAKE_CURRENT_VOL="netork_postgres17_data")
assert result.returncode == 0, result.stderr
assert any("up -d --no-deps --wait" in c and c.endswith(" postgres") for c in calls)
assert not any(" stop " in c for c in calls)
assert not any("pg_dump" in c for c in calls)
def test_a_new_volume_stops_a_normal_deploy_before_anything_changed(tmp_path: Path) -> None:
result, calls = _run(tmp_path)
assert result.returncode == 3
assert "--infra-db" in result.stderr
assert not any(" up " in c or " stop " in c or "pg_dump" in c for c in calls)
def test_infra_db_moves_the_data_by_dump_and_restore(tmp_path: Path) -> None:
result, calls = _run(tmp_path, mode="move")
assert result.returncode == 0, result.stderr
stop = _first(calls, " stop netork-api netork-worker netork-beat")
dump = _first(calls, "pg_dump")
up = _first(calls, " up -d --no-deps --wait")
restore = _first(calls, "pg_restore")
assert stop < dump < up < restore
assert any(c.endswith("ANALYZE") for c in calls)
assert (tmp_path / "restored").read_text() == "DUMPDATA\n"
dumps = list((tmp_path / "netork" / "backups").glob("db-netork_postgres_data-*.dump"))
assert len(dumps) == 1, "the dump must stay on the host"
assert "netork_postgres_data" in result.stdout # the old volume is named as the way back
def test_counts_that_differ_fail_the_move_and_say_how_back(tmp_path: Path) -> None:
result, _ = _run(tmp_path, mode="move", FAKE_COUNTS_AFTER="public.t=4")
assert result.returncode != 0
assert "--infra-db" in result.stderr and "netork_postgres_data" in result.stderr
def test_another_revision_fails_the_move(tmp_path: Path) -> None:
result, _ = _run(tmp_path, mode="move", FAKE_REV_AFTER="other")
assert result.returncode != 0
def test_a_failed_restore_fails_the_move(tmp_path: Path) -> None:
result, _ = _run(tmp_path, mode="move", FAKE_RESTORE_EXIT="1")
assert result.returncode != 0
def test_an_existing_volume_is_switched_onto_without_a_restore(tmp_path: Path) -> None:
"""The way back: compose names the old volume again, and it is still there."""
result, calls = _run(
tmp_path,
mode="move",
FAKE_CURRENT_VOL="netork_postgres17_data",
FAKE_CONFIG=_config("postgres_data", "netork_postgres_data"),
FAKE_TARGET_EXISTS="1",
)
assert result.returncode == 0, result.stderr
assert _first(calls, " stop netork-api") < _first(calls, " up -d --no-deps --wait")
assert not any("pg_dump" in c or "pg_restore" in c for c in calls)
def test_the_counts_compared_are_real(tmp_path: Path) -> None:
"""Guards the fake: equal counts must come from rows, not from two empty lists."""
result, _ = _run(tmp_path, mode="move")
assert result.returncode == 0, result.stderr
before = next((tmp_path / "netork" / "backups").glob("db-*.before")).read_text()
assert before == "public.t=5\n"