CI / check (pull_request) Failing after 28s
remote/database.sh runs on the host (over ssh's stdin) after .env and before broker and engine: - no bundled database: nothing to do; - the volume it already uses: plain `up -d --wait`, recreated only on a changed image or definition, so a new PostgreSQL minor release arrives with the next deploy; - another volume (a new major version): a normal deploy stops there, nothing changed; with --infra-db the engine stops, the old database is counted and dumped (kept in ~/netork/backups), the new container comes up on the new volume, restore, ANALYZE, counts and Alembic revision must match. The old volume stays, and deploying the previous version with --infra-db switches back onto it without a restore. Checked on .50 in reconcile mode: the running postgres:16 container stayed as it was. README describes the steps and the way back.
205 lines
7.2 KiB
Python
205 lines
7.2 KiB
Python
"""The bundled PostgreSQL on a deploy (NetOrk/netork#1308): remote/database.sh.
|
|
|
|
deploy.sh runs it on the host before the broker and the engine:
|
|
|
|
- no bundled database (external PostgreSQL): nothing to do;
|
|
- compose names the volume the container already uses: a plain `up -d --wait`,
|
|
which recreates the container only when its image or definition changed, so
|
|
a new PostgreSQL minor release reaches the host with the next deploy;
|
|
- compose names another volume (a new major version): a normal deploy stops
|
|
there, before anything changed, and points at --infra-db;
|
|
- with --infra-db and a volume that does not exist yet: the engine is stopped,
|
|
the old database is counted and dumped (the dump stays in ~/netork/backups),
|
|
the new container comes up on the new volume, the dump is restored, ANALYZE,
|
|
and the counts and the Alembic revision must match. The old volume stays;
|
|
- with --infra-db and a volume that exists (the way back): the engine is
|
|
stopped and the container switches onto it, without a restore.
|
|
|
|
The remote side is a fake `docker` that plays the host and records each call.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
SCRIPT = Path(__file__).resolve().parent.parent / "remote" / "database.sh"
|
|
ENGINE = ["netork-api", "netork-worker", "netork-beat"]
|
|
|
|
_FAKE_DOCKER = r"""#!/bin/bash
|
|
state=$FAKE_STATE
|
|
printf '%s\n' "$*" >> "$state/calls"
|
|
upped() { [ -f "$state/upped" ]; }
|
|
case "$1" in
|
|
ps)
|
|
[ -n "$FAKE_PG_CID" ] || exit 0
|
|
if upped; then echo newcid; else echo "$FAKE_PG_CID"; fi ;;
|
|
inspect) echo "$FAKE_CURRENT_VOL" ;;
|
|
volume) [ "$FAKE_TARGET_EXISTS" = 1 ] && exit 0 || exit 1 ;;
|
|
start) exit 0 ;;
|
|
compose)
|
|
case "$*" in
|
|
*" config "*) printf '%s' "$FAKE_CONFIG" ;;
|
|
*" up -d "*) touch "$state/upped" ;;
|
|
esac ;;
|
|
exec)
|
|
shift
|
|
[ "$1" = -i ] && shift
|
|
shift # container
|
|
case "$1" in
|
|
printenv) echo netork ;;
|
|
pg_isready) exit 0 ;;
|
|
pg_dump) echo DUMPDATA ;;
|
|
pg_restore) cat > "$state/restored"; exit "${FAKE_RESTORE_EXIT:-0}" ;;
|
|
psql)
|
|
q="${*: -1}"
|
|
case "$q" in
|
|
*alembic_version*) if upped; then echo "${FAKE_REV_AFTER:-abc}"; else echo abc; fi ;;
|
|
*string_agg*) echo "SELECT 'public.t' AS t, 1 AS n" ;;
|
|
ANALYZE) ;;
|
|
*"FROM (SELECT 'public.t'"*) if upped; then echo "${FAKE_COUNTS_AFTER:-public.t=5}"; else echo "public.t=5"; fi ;;
|
|
*pg_database_size*) echo 1000 ;;
|
|
esac ;;
|
|
esac ;;
|
|
esac
|
|
exit 0
|
|
"""
|
|
|
|
|
|
def _config(source: str, name: str) -> str:
|
|
return json.dumps(
|
|
{
|
|
"services": {
|
|
"postgres": {
|
|
"volumes": [
|
|
{"type": "volume", "source": source, "target": "/var/lib/postgresql/data"}
|
|
]
|
|
}
|
|
},
|
|
"volumes": {source: {"name": name}},
|
|
},
|
|
indent=2,
|
|
)
|
|
|
|
|
|
def _run(tmp_path: Path, mode: str = "reconcile", **fake: str):
|
|
bin_dir = tmp_path / "bin"
|
|
bin_dir.mkdir()
|
|
docker = bin_dir / "docker"
|
|
docker.write_text(_FAKE_DOCKER)
|
|
docker.chmod(0o755)
|
|
(tmp_path / "netork").mkdir()
|
|
env = {
|
|
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
|
"HOME": str(tmp_path),
|
|
"FAKE_STATE": str(tmp_path),
|
|
"FAKE_PG_CID": "oldcid",
|
|
"FAKE_CURRENT_VOL": "netork_postgres_data",
|
|
"FAKE_CONFIG": _config("postgres17_data", "netork_postgres17_data"),
|
|
"FAKE_TARGET_EXISTS": "0",
|
|
"REGISTRY_HOST": "registry.example",
|
|
"NETORK_VERSION": "latest-dev",
|
|
**fake,
|
|
}
|
|
result = subprocess.run(
|
|
["bash", str(SCRIPT), mode, *ENGINE],
|
|
env=env,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
check=False,
|
|
)
|
|
calls_file = tmp_path / "calls"
|
|
calls = calls_file.read_text().splitlines() if calls_file.exists() else []
|
|
return result, calls
|
|
|
|
|
|
def _first(calls: list[str], needle: str) -> int:
|
|
return next(i for i, c in enumerate(calls) if needle in c)
|
|
|
|
|
|
def test_an_external_database_is_left_alone(tmp_path: Path) -> None:
|
|
result, calls = _run(tmp_path, FAKE_PG_CID="")
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert not any(c.startswith("compose") for c in calls)
|
|
assert "external" in result.stdout
|
|
|
|
|
|
def test_the_same_volume_gets_a_plain_reconcile(tmp_path: Path) -> None:
|
|
result, calls = _run(tmp_path, FAKE_CURRENT_VOL="netork_postgres17_data")
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert any("up -d --no-deps --wait" in c and c.endswith(" postgres") for c in calls)
|
|
assert not any(" stop " in c for c in calls)
|
|
assert not any("pg_dump" in c for c in calls)
|
|
|
|
|
|
def test_a_new_volume_stops_a_normal_deploy_before_anything_changed(tmp_path: Path) -> None:
|
|
result, calls = _run(tmp_path)
|
|
|
|
assert result.returncode == 3
|
|
assert "--infra-db" in result.stderr
|
|
assert not any(" up " in c or " stop " in c or "pg_dump" in c for c in calls)
|
|
|
|
|
|
def test_infra_db_moves_the_data_by_dump_and_restore(tmp_path: Path) -> None:
|
|
result, calls = _run(tmp_path, mode="move")
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
stop = _first(calls, " stop netork-api netork-worker netork-beat")
|
|
dump = _first(calls, "pg_dump")
|
|
up = _first(calls, " up -d --no-deps --wait")
|
|
restore = _first(calls, "pg_restore")
|
|
assert stop < dump < up < restore
|
|
assert any(c.endswith("ANALYZE") for c in calls)
|
|
assert (tmp_path / "restored").read_text() == "DUMPDATA\n"
|
|
dumps = list((tmp_path / "netork" / "backups").glob("db-netork_postgres_data-*.dump"))
|
|
assert len(dumps) == 1, "the dump must stay on the host"
|
|
assert "netork_postgres_data" in result.stdout # the old volume is named as the way back
|
|
|
|
|
|
def test_counts_that_differ_fail_the_move_and_say_how_back(tmp_path: Path) -> None:
|
|
result, _ = _run(tmp_path, mode="move", FAKE_COUNTS_AFTER="public.t=4")
|
|
|
|
assert result.returncode != 0
|
|
assert "--infra-db" in result.stderr and "netork_postgres_data" in result.stderr
|
|
|
|
|
|
def test_another_revision_fails_the_move(tmp_path: Path) -> None:
|
|
result, _ = _run(tmp_path, mode="move", FAKE_REV_AFTER="other")
|
|
|
|
assert result.returncode != 0
|
|
|
|
|
|
def test_a_failed_restore_fails_the_move(tmp_path: Path) -> None:
|
|
result, _ = _run(tmp_path, mode="move", FAKE_RESTORE_EXIT="1")
|
|
|
|
assert result.returncode != 0
|
|
|
|
|
|
def test_an_existing_volume_is_switched_onto_without_a_restore(tmp_path: Path) -> None:
|
|
"""The way back: compose names the old volume again, and it is still there."""
|
|
result, calls = _run(
|
|
tmp_path,
|
|
mode="move",
|
|
FAKE_CURRENT_VOL="netork_postgres17_data",
|
|
FAKE_CONFIG=_config("postgres_data", "netork_postgres_data"),
|
|
FAKE_TARGET_EXISTS="1",
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert _first(calls, " stop netork-api") < _first(calls, " up -d --no-deps --wait")
|
|
assert not any("pg_dump" in c or "pg_restore" in c for c in calls)
|
|
|
|
|
|
def test_the_counts_compared_are_real(tmp_path: Path) -> None:
|
|
"""Guards the fake: equal counts must come from rows, not from two empty lists."""
|
|
result, _ = _run(tmp_path, mode="move")
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
before = next((tmp_path / "netork" / "backups").glob("db-*.before")).read_text()
|
|
assert before == "public.t=5\n"
|