From 3a15c311a08cbfb3ed76b5775d6a0850bfc8655b Mon Sep 17 00:00:00 2001 From: Christian Manivong Date: Sun, 18 Oct 2026 18:26:24 +0200 Subject: [PATCH] fix: record the version in .env before starting the containers The services read ~/netork/.env through env_file, so whatever it says when `up -d` creates a container ends up in that container's environment. The settings were recorded only after the swap, so every container carried the previous deploy's NETORK_VERSION: the image was right, `docker exec env` was not. Record them once the pull has brought the images onto the host and before the first `up -d`. A failed pull still leaves .env alone. If the swap fails afterwards, .env names the tag whose compose files and images are already on the host, which is what a hand-typed `docker compose up -d` there should start. Also say which branches a feature- tag comes from: feature/ and feat/ become feature-, and fix/ now becomes feature-fix- (NetOrk/netork#208), because the registry keeps only feature-* branch tags. The accepted forms do not change; the comment, the refusal message and the README now say so, and a test holds that feature-fix- passes and a bare fix- does not. Closes #6 --- README.md | 29 ++++--- deploy.sh | 75 +++++++++-------- tests/test_deploy_env_before_up.py | 102 +++++++++++++++++++++++ tests/test_deploy_script_fails_loudly.py | 29 ++++--- tests/test_deploy_versions.py | 91 ++++++++++++++++++++ 5 files changed, 269 insertions(+), 57 deletions(-) create mode 100644 tests/test_deploy_env_before_up.py create mode 100644 tests/test_deploy_versions.py diff --git a/README.md b/README.md index 174da74..19b6dd9 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ Only tags netOrk's CI publishes are accepted: - `latest` - `latest-dev` - `main-` -- `feature-` +- `feature-`, a branch build: `feature/` and `feat/` become + `feature-`, and `fix/` becomes `feature-fix-`. Any further `/` in + the branch name becomes `-` as well. - `X.Y.Z` Anything else is refused before any server is touched. @@ -90,22 +92,25 @@ started earlier pulls whatever image the registry held before, which is stale. If the registry's answer can't be read, the pull below decides. 2. Pulls the engine image and copies `docker-compose.yml` and `docker-compose.registry.yml` out of it into `~/netork/`. -3. Pulls the netOrk images, then force-recreates the API, the workers, `netork-beat`, - `flower` and, on UI servers, `netork-ui`. If the recreate fails, it is tried once - more after 5 s (`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose - a container it just renamed and leave the rest stopped. If the second attempt fails - too, the container states are printed and the deploy fails. -4. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its +3. Pulls the netOrk images. +4. Records `REGISTRY_HOST` and `NETORK_VERSION` in `~/netork/.env`, so that a + hand-typed `docker compose` on the host uses the same images. It does this before + any container starts, because the services read `.env` into their environment. + A failed pull leaves `.env` as it was. +5. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers, + `netork-ui`. If the recreate fails, it is tried once more after 5 s + (`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it + just renamed and leave the rest stopped. If the second attempt fails too, the + container states are printed and the deploy fails. +6. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its definition changed. It never touches `postgres` or `redis`. It then removes services netOrk no longer ships, container and volume, where a host still has them: today the bundled `registry` (NetOrk/netork#763). It reports only what it actually removed; on a host that was cleaned earlier it says nothing. -5. Verifies that the containers run exactly the image that was pulled. If they don't, the +7. Verifies that the containers run exactly the image that was pulled. If they don't, the deploy fails. -6. Records `REGISTRY_HOST` and `NETORK_VERSION` in `~/netork/.env`, so that a - hand-typed `docker compose` on the host uses the same images. -7. Runs `alembic upgrade head` inside `netork-api`. -8. Prunes unused images. +8. Runs `alembic upgrade head` inside `netork-api`. +9. Prunes unused images. A failing step stops that server's deploy with a non-zero exit, and the other servers carry on. The script exits non-zero if any server failed and names those servers. diff --git a/deploy.sh b/deploy.sh index 43fa5e2..0861688 100644 --- a/deploy.sh +++ b/deploy.sh @@ -208,15 +208,18 @@ fi # silently, with the only symptom surfacing later at `alembic upgrade head`, # by which point containers are already switched over. Reject anything that # doesn't match a tag format netOrk's CI actually publishes: latest, -# latest-dev, main-, feature- (feature/* only), or a release -# version (X.Y.Z). See NetOrk/netork#55. +# latest-dev, main-, feature-, or a release version (X.Y.Z). +# Every branch build is a feature- tag: feature/ and feat/ +# become feature-, fix/ becomes feature-fix-. The registry +# keeps feature-* tags for 14 days; a tag of any other form would be deleted +# within a day. See NetOrk/netork#55 and NetOrk/netork#208. VALID_VERSION_RE='^(latest|latest-dev|main-[0-9a-f]{7,40}|feature-[A-Za-z0-9._-]+|[0-9]+\.[0-9]+\.[0-9]+)$' _validate_version() { local version="$1" if [[ ! "$version" =~ $VALID_VERSION_RE ]]; then echo "Refusing to deploy version '${version}': not a tag CI publishes (expected latest," \ - "latest-dev, main-, feature-, or X.Y.Z)." >&2 + "latest-dev, main-, feature- (also for fix/* branches), or X.Y.Z)." >&2 return 1 fi } @@ -313,6 +316,42 @@ deploy_steps() { REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}" + # docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project + # .env when they are not in the environment. This script passes both inline, + # but a hand-typed `docker compose up -d flower` on the host needs them there + # or dies on "invalid reference format". Values are rewritten in place, so + # the file does not grow a new line per deploy. + # + # Not earlier: this used to run before the pull, so a deploy that failed on a + # tag the registry does not have still left the host recording that tag, and + # the next `docker compose up` a human ran there reached for an image that + # does not exist. Once the pull went through, the images are on the host. + # + # Not later: the services also read this file (`env_file: .env`), so it is in + # every container's environment. Written after the swap, as it was for a + # while, it made each container carry the previous deploy's NETORK_VERSION + # (NetOrk/deploy#6). Should the swap below fail, the file names the tag whose + # compose files and images are already on the host, which is what a + # hand-typed `docker compose up -d` there should start. + echo "[${SERVER}] Recording registry settings in ~/netork/.env..." + # shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely + ssh "$SERVER" "bash -s" <> .env +for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do + key=\${kv%%=*} + if grep -q "^\${key}=" .env; then + sed -i "s|^\${key}=.*|\${kv}|" .env + else + printf '%s\n' "\${kv}" >> .env + fi +done +REMOTE_ENV + # --force-recreate is deliberate: `docker compose up -d` decides what to # replace from the service definition's config hash, and the image reference # (`.../ui:latest-dev`) does not change when the tag is moved to a new image. @@ -417,36 +456,6 @@ deploy_steps() { fi echo "[${SERVER}] Verified: containers run ${SERVER_VERSION}." - # docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project - # .env when they are not in the environment. This script passes both inline, - # but a hand-typed `docker compose up -d flower` on the host needs them there - # or dies on "invalid reference format". Values are rewritten in place, so - # the file does not grow a new line per deploy. - # - # Only now. This used to run before the pull, so a deploy that failed on a tag - # the registry does not have still left the host recording that tag — and the - # next `docker compose up` a human ran there reached for an image that does - # not exist. The file is a record of what this server runs, so it is written - # once that is true: after the swap and after the image-id check agreed. - echo "[${SERVER}] Recording registry settings in ~/netork/.env..." - # shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely - ssh "$SERVER" "bash -s" <> .env -for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do - key=\${kv%%=*} - if grep -q "^\${key}=" .env; then - sed -i "s|^\${key}=.*|\${kv}|" .env - else - printf '%s\n' "\${kv}" >> .env - fi -done -REMOTE_ENV - # netOrk migrates *after* the swap — its API tolerates the gap. What it does # not tolerate is the migration failing unnoticed: the new code is already # serving, so a missing column is a 500 on every request that touches it diff --git a/tests/test_deploy_env_before_up.py b/tests/test_deploy_env_before_up.py new file mode 100644 index 0000000..2d0fb83 --- /dev/null +++ b/tests/test_deploy_env_before_up.py @@ -0,0 +1,102 @@ +"""The containers start with the version being deployed in `~/netork/.env` (NetOrk/deploy#6). + +The services read `~/netork/.env` through `env_file: .env`, so whatever the file +says when `docker compose up -d` creates a container ends up in that container's +environment. The script used to record `NETORK_VERSION` there only after the +containers had been started, so every container carried the *previous* deploy's +version: a host moved from a feature tag to `latest-dev` ran the `latest-dev` +image (the tag comes from the inline variable) but `docker exec env` still +said `NETORK_VERSION=feature-container-engine-api`. + +The remote side is a fake `ssh`. The `.env` step (`bash -s` with a heredoc) runs +for real against a fake home directory; every `up -d` notes what `.env` says at +that moment. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" + +_FAKE_SSH = """#!/bin/sh +for arg in "$@"; do cmd="$arg"; done +# The host lock's holder runs for real, here (test_deploy_host_lock.py). +case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac +# The .env step runs for real, in the fake home. +[ "$cmd" = "bash -s" ] && exec bash -s +cat > /dev/null +case "$cmd" in + *" pull "*) + if [ -n "{fail_pull}" ]; then echo "Error: manifest unknown"; exit 1; fi + ;; + *"up -d"*) + grep '^NETORK_VERSION=' "$HOME/netork/.env" >> "{state}/env-at-up" || echo none >> "{state}/env-at-up" + ;; +esac +exit 0 +""" + +PREVIOUS = "NETORK_VERSION=feature-previous" + + +def _deploy(tmp_path: Path, *, fail_pull: bool = False) -> subprocess.CompletedProcess[str]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake = bin_dir / "ssh" + fake.write_text( + _FAKE_SSH.replace("{state}", str(tmp_path)).replace("{fail_pull}", "x" if fail_pull else "") + ) + fake.chmod(0o755) + netork = tmp_path / "netork" + netork.mkdir() + (netork / ".env").write_text( + f"POSTGRES_PASSWORD=secret\nREGISTRY_HOST=old.example\n{PREVIOUS}\n" + ) + return subprocess.run( + ["bash", str(DEPLOY), "--version=latest-dev", "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + +def test_every_up_sees_the_version_being_deployed(tmp_path: Path) -> None: + result = _deploy(tmp_path) + + assert result.returncode == 0, result.stderr + seen = (tmp_path / "env-at-up").read_text().splitlines() + # The engine services' recreate, then the infrastructure reconcile. + assert len(seen) == 2, seen + assert seen == ["NETORK_VERSION=latest-dev"] * 2, ( + "a container was started while .env still named the previous deploy's version" + ) + + +def test_the_record_keeps_the_rest_of_env(tmp_path: Path) -> None: + result = _deploy(tmp_path) + + assert result.returncode == 0, result.stderr + assert (tmp_path / "netork" / ".env").read_text() == ( + "POSTGRES_PASSWORD=secret\nREGISTRY_HOST=registry.example\nNETORK_VERSION=latest-dev\n" + ) + + +def test_a_failed_pull_leaves_env_alone(tmp_path: Path) -> None: + """A tag the registry does not have must not end up recorded on the host + (the reason the step once moved behind the image check).""" + result = _deploy(tmp_path, fail_pull=True) + + assert result.returncode != 0 + assert PREVIOUS in (tmp_path / "netork" / ".env").read_text() + assert not (tmp_path / "env-at-up").exists(), "containers were started after a failed pull" diff --git a/tests/test_deploy_script_fails_loudly.py b/tests/test_deploy_script_fails_loudly.py index d77c5ea..d727108 100644 --- a/tests/test_deploy_script_fails_loudly.py +++ b/tests/test_deploy_script_fails_loudly.py @@ -107,9 +107,13 @@ class TestTheHostRecordsOnlyWhatItRuns: failure path above: the deploy stopped exactly where it should, and still left `NETORK_VERSION=main-0000000` behind. - So it is written after the swap and after the image-id check agreed, and this - holds that order. Positional rather than behavioural, which is the honest - limit of a static check — but the order is the whole property. + It then moved behind the image check, and that was too late the other way: + the services read the file through `env_file: .env`, so every container was + started with the previous deploy's version in its environment + (NetOrk/deploy#6). So it is written once the pull has brought the images + onto the host, and before the first `up -d`. Positional rather than + behavioural, which is the honest limit of a static check; the behaviour is + in test_deploy_env_before_up.py. """ @staticmethod @@ -118,14 +122,15 @@ class TestTheHostRecordsOnlyWhatItRuns: assert text.count(needle) == 1, f"{needle!r} appears {text.count(needle)} times" return text.index(needle) - def test_env_is_recorded_after_the_image_check(self) -> None: - verified = self._position("Verified: containers run") - recorded = self._position("Recording registry settings") - - assert recorded > verified, ( - "the host records NETORK_VERSION before the deploy is verified; " - "a failed deploy then leaves it pointing at a version never run" - ) - def test_env_is_recorded_after_the_pull(self) -> None: assert self._position("Recording registry settings") > self._position("Pulling images...") + + def test_env_is_recorded_before_the_first_up(self) -> None: + lines = _logical_lines() + first_up = min(i for i, line in enumerate(lines) if "up -d" in line) + recorded = next(i for i, line in enumerate(lines) if "Recording registry settings" in line) + + assert recorded < first_up, ( + "containers are started before .env records NETORK_VERSION; " + "they would carry the previous deploy's version in their environment" + ) diff --git a/tests/test_deploy_versions.py b/tests/test_deploy_versions.py new file mode 100644 index 0000000..47ebe55 --- /dev/null +++ b/tests/test_deploy_versions.py @@ -0,0 +1,91 @@ +"""`--version` takes only the tag formats netOrk's CI publishes (NetOrk/netork#55, #208). + +Every branch build is a `feature-` tag, with `/` turned into `-`: `feature/x` and +`feat/x` become `feature-x`, `fix/x` becomes `feature-fix-x`. The registry keeps +`feature-*` tags for 14 days and deletes a tag of any other form within a day, so +a bare `fix-x` is refused like any other unknown form. Anything else is refused +before a host is touched, so a typo cannot pull some ancient leftover tag. + +The fake `ssh` only notes that it was reached and fails, which ends the deploy +right after the validation. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh" + + +def _deploy(tmp_path: Path, version: str) -> tuple[subprocess.CompletedProcess[str], bool]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + marker = tmp_path / "ssh-was-called" + fake = bin_dir / "ssh" + fake.write_text(f"#!/bin/sh\ntouch {marker}\nexit 255\n") + fake.chmod(0o755) + result = subprocess.run( + ["bash", str(DEPLOY), f"--version={version}", "testhost"], + env={ + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "HOME": str(tmp_path), + "DEPLOY_ENV_FILE": "/dev/null", + "REGISTRY_HOST": "registry.example", + }, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + return result, marker.exists() + + +@pytest.mark.parametrize( + "version", + [ + "latest", + "latest-dev", + "main-1a2b3c4", + "feature-container-engine-api", + "feature-fix-a-hook-below-the-early-returns", + "feature-fix-ui-warning-icons", + "0.37.0", + ], +) +def test_a_published_tag_is_accepted(tmp_path: Path, version: str) -> None: + result, reached_ssh = _deploy(tmp_path, version) + + assert "Refusing to deploy version" not in result.stderr + assert reached_ssh + + +@pytest.mark.parametrize( + "version", + [ + "bogus", + "fix-a-hook", + "fix/a-hook", + "feature-", + "feature-fix/ui/warning-icons", + "feature-fix-a;id", + "feature-$(id)", + "main-xyz1234", + ], +) +def test_anything_else_is_refused_before_a_host_is_touched(tmp_path: Path, version: str) -> None: + result, reached_ssh = _deploy(tmp_path, version) + + assert result.returncode != 0 + assert f"Refusing to deploy version '{version}'" in result.stderr + assert not reached_ssh + + +def test_the_refusal_names_branch_builds(tmp_path: Path) -> None: + result, _ = _deploy(tmp_path, "bogus") + + assert "feature- (also for fix/* branches)" in result.stderr