Merge pull request 'fix: record the version in .env before starting the containers' (#11) from fix/env-before-up-6 into main
CI / check (push) Successful in 20s

This commit was merged in pull request #11.
This commit is contained in:
christianmanivong committed 2026-10-18 16:37:17 +00:00
commit c1b1eb07b9
5 files changed
+269 -57

No files matched your search

+17 -12
View File
@@ -67,7 +67,9 @@ Only tags netOrk's CI publishes are accepted:
- `latest`
- `latest-dev`
- `main-<sha>`
- `feature-<branch>`
- `feature-<name>`, a branch build: `feature/<name>` and `feat/<name>` become
`feature-<name>`, and `fix/<name>` becomes `feature-fix-<name>`. Any further `/` in
the branch name becomes `-` as well.
- `X.Y.Z`
Anything else is refused before any server is touched.
@@ -90,22 +92,25 @@ started earlier pulls whatever image the registry held before, which is stale.
If the registry's answer can't be read, the pull below decides.
2. Pulls the engine image and copies `docker-compose.yml` and
`docker-compose.registry.yml` out of it into `~/netork/`.
3. Pulls the netOrk images, then force-recreates the API, the workers, `netork-beat`,
`flower` and, on UI servers, `netork-ui`. If the recreate fails, it is tried once
more after 5 s (`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose
a container it just renamed and leave the rest stopped. If the second attempt fails
too, the container states are printed and the deploy fails.
4. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
3. Pulls the netOrk images.
4. Records `REGISTRY_HOST` and `NETORK_VERSION` in `~/netork/.env`, so that a
hand-typed `docker compose` on the host uses the same images. It does this before
any container starts, because the services read `.env` into their environment.
A failed pull leaves `.env` as it was.
5. Force-recreates the API, the workers, `netork-beat`, `flower` and, on UI servers,
`netork-ui`. If the recreate fails, it is tried once more after 5 s
(`DEPLOY_RECREATE_RETRY_DELAY`). Compose's parallel recreate can lose a container it
just renamed and leave the rest stopped. If the second attempt fails too, the
container states are printed and the deploy fails.
6. Reconciles `apt-cacher-ng` and `signal-api`: each is recreated only if its
definition changed. It never touches `postgres` or `redis`.
It then removes services netOrk no longer ships, container and volume, where a host
still has them: today the bundled `registry` (NetOrk/netork#763). It reports only
what it actually removed; on a host that was cleaned earlier it says nothing.
5. Verifies that the containers run exactly the image that was pulled. If they don't, the
7. Verifies that the containers run exactly the image that was pulled. If they don't, the
deploy fails.
6. Records `REGISTRY_HOST` and `NETORK_VERSION` in `~/netork/.env`, so that a
hand-typed `docker compose` on the host uses the same images.
7. Runs `alembic upgrade head` inside `netork-api`.
8. Prunes unused images.
8. Runs `alembic upgrade head` inside `netork-api`.
9. Prunes unused images.
A failing step stops that server's deploy with a non-zero exit, and the other servers
carry on. The script exits non-zero if any server failed and names those servers.
+42 -33
View File
@@ -208,15 +208,18 @@ fi
# silently, with the only symptom surfacing later at `alembic upgrade head`,
# by which point containers are already switched over. Reject anything that
# doesn't match a tag format netOrk's CI actually publishes: latest,
# latest-dev, main-<sha>, feature-<branch> (feature/* only), or a release
# version (X.Y.Z). See NetOrk/netork#55.
# latest-dev, main-<sha>, feature-<branch>, or a release version (X.Y.Z).
# Every branch build is a feature-<branch> tag: feature/<name> and feat/<name>
# become feature-<name>, fix/<name> becomes feature-fix-<name>. The registry
# keeps feature-* tags for 14 days; a tag of any other form would be deleted
# within a day. See NetOrk/netork#55 and NetOrk/netork#208.
VALID_VERSION_RE='^(latest|latest-dev|main-[0-9a-f]{7,40}|feature-[A-Za-z0-9._-]+|[0-9]+\.[0-9]+\.[0-9]+)$'
_validate_version() {
local version="$1"
if [[ ! "$version" =~ $VALID_VERSION_RE ]]; then
echo "Refusing to deploy version '${version}': not a tag CI publishes (expected latest," \
"latest-dev, main-<sha>, feature-<branch>, or X.Y.Z)." >&2
"latest-dev, main-<sha>, feature-<branch> (also for fix/* branches), or X.Y.Z)." >&2
return 1
fi
}
@@ -313,6 +316,42 @@ deploy_steps() {
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}"
# docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project
# .env when they are not in the environment. This script passes both inline,
# but a hand-typed `docker compose up -d flower` on the host needs them there
# or dies on "invalid reference format". Values are rewritten in place, so
# the file does not grow a new line per deploy.
#
# Not earlier: this used to run before the pull, so a deploy that failed on a
# tag the registry does not have still left the host recording that tag, and
# the next `docker compose up` a human ran there reached for an image that
# does not exist. Once the pull went through, the images are on the host.
#
# Not later: the services also read this file (`env_file: .env`), so it is in
# every container's environment. Written after the swap, as it was for a
# while, it made each container carry the previous deploy's NETORK_VERSION
# (NetOrk/deploy#6). Should the swap below fail, the file names the tag whose
# compose files and images are already on the host, which is what a
# hand-typed `docker compose up -d` there should start.
echo "[${SERVER}] Recording registry settings in ~/netork/.env..."
# shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely
ssh "$SERVER" "bash -s" <<REMOTE_ENV
set -eu
cd ~/netork
touch .env
# Append to a file that does not end in a newline, and the last line grows a
# suffix instead of gaining a neighbour.
[ ! -s .env ] || [ -z "\$(tail -c1 .env)" ] || printf '\n' >> .env
for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do
key=\${kv%%=*}
if grep -q "^\${key}=" .env; then
sed -i "s|^\${key}=.*|\${kv}|" .env
else
printf '%s\n' "\${kv}" >> .env
fi
done
REMOTE_ENV
# --force-recreate is deliberate: `docker compose up -d` decides what to
# replace from the service definition's config hash, and the image reference
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
@@ -417,36 +456,6 @@ deploy_steps() {
fi
echo "[${SERVER}] Verified: containers run ${SERVER_VERSION}."
# docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project
# .env when they are not in the environment. This script passes both inline,
# but a hand-typed `docker compose up -d flower` on the host needs them there
# or dies on "invalid reference format". Values are rewritten in place, so
# the file does not grow a new line per deploy.
#
# Only now. This used to run before the pull, so a deploy that failed on a tag
# the registry does not have still left the host recording that tag — and the
# next `docker compose up` a human ran there reached for an image that does
# not exist. The file is a record of what this server runs, so it is written
# once that is true: after the swap and after the image-id check agreed.
echo "[${SERVER}] Recording registry settings in ~/netork/.env..."
# shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely
ssh "$SERVER" "bash -s" <<REMOTE_ENV
set -eu
cd ~/netork
touch .env
# Append to a file that does not end in a newline, and the last line grows a
# suffix instead of gaining a neighbour.
[ ! -s .env ] || [ -z "\$(tail -c1 .env)" ] || printf '\n' >> .env
for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do
key=\${kv%%=*}
if grep -q "^\${key}=" .env; then
sed -i "s|^\${key}=.*|\${kv}|" .env
else
printf '%s\n' "\${kv}" >> .env
fi
done
REMOTE_ENV
# netOrk migrates *after* the swap — its API tolerates the gap. What it does
# not tolerate is the migration failing unnoticed: the new code is already
# serving, so a missing column is a 500 on every request that touches it
+102
View File
@@ -0,0 +1,102 @@
"""The containers start with the version being deployed in `~/netork/.env` (NetOrk/deploy#6).
The services read `~/netork/.env` through `env_file: .env`, so whatever the file
says when `docker compose up -d` creates a container ends up in that container's
environment. The script used to record `NETORK_VERSION` there only after the
containers had been started, so every container carried the *previous* deploy's
version: a host moved from a feature tag to `latest-dev` ran the `latest-dev`
image (the tag comes from the inline variable) but `docker exec <api> env` still
said `NETORK_VERSION=feature-container-engine-api`.
The remote side is a fake `ssh`. The `.env` step (`bash -s` with a heredoc) runs
for real against a fake home directory; every `up -d` notes what `.env` says at
that moment.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
_FAKE_SSH = """#!/bin/sh
for arg in "$@"; do cmd="$arg"; done
# The host lock's holder runs for real, here (test_deploy_host_lock.py).
case "$cmd" in *.deploy.lock*) exec sh -c "$cmd" ;; esac
# The .env step runs for real, in the fake home.
[ "$cmd" = "bash -s" ] && exec bash -s
cat > /dev/null
case "$cmd" in
*" pull "*)
if [ -n "{fail_pull}" ]; then echo "Error: manifest unknown"; exit 1; fi
;;
*"up -d"*)
grep '^NETORK_VERSION=' "$HOME/netork/.env" >> "{state}/env-at-up" || echo none >> "{state}/env-at-up"
;;
esac
exit 0
"""
PREVIOUS = "NETORK_VERSION=feature-previous"
def _deploy(tmp_path: Path, *, fail_pull: bool = False) -> subprocess.CompletedProcess[str]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
fake = bin_dir / "ssh"
fake.write_text(
_FAKE_SSH.replace("{state}", str(tmp_path)).replace("{fail_pull}", "x" if fail_pull else "")
)
fake.chmod(0o755)
netork = tmp_path / "netork"
netork.mkdir()
(netork / ".env").write_text(
f"POSTGRES_PASSWORD=secret\nREGISTRY_HOST=old.example\n{PREVIOUS}\n"
)
return subprocess.run(
["bash", str(DEPLOY), "--version=latest-dev", "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=60,
check=False,
)
def test_every_up_sees_the_version_being_deployed(tmp_path: Path) -> None:
result = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
seen = (tmp_path / "env-at-up").read_text().splitlines()
# The engine services' recreate, then the infrastructure reconcile.
assert len(seen) == 2, seen
assert seen == ["NETORK_VERSION=latest-dev"] * 2, (
"a container was started while .env still named the previous deploy's version"
)
def test_the_record_keeps_the_rest_of_env(tmp_path: Path) -> None:
result = _deploy(tmp_path)
assert result.returncode == 0, result.stderr
assert (tmp_path / "netork" / ".env").read_text() == (
"POSTGRES_PASSWORD=secret\nREGISTRY_HOST=registry.example\nNETORK_VERSION=latest-dev\n"
)
def test_a_failed_pull_leaves_env_alone(tmp_path: Path) -> None:
"""A tag the registry does not have must not end up recorded on the host
(the reason the step once moved behind the image check)."""
result = _deploy(tmp_path, fail_pull=True)
assert result.returncode != 0
assert PREVIOUS in (tmp_path / "netork" / ".env").read_text()
assert not (tmp_path / "env-at-up").exists(), "containers were started after a failed pull"
+17 -12
View File
@@ -107,9 +107,13 @@ class TestTheHostRecordsOnlyWhatItRuns:
failure path above: the deploy stopped exactly where it should, and still
left `NETORK_VERSION=main-0000000` behind.
So it is written after the swap and after the image-id check agreed, and this
holds that order. Positional rather than behavioural, which is the honest
limit of a static check — but the order is the whole property.
It then moved behind the image check, and that was too late the other way:
the services read the file through `env_file: .env`, so every container was
started with the previous deploy's version in its environment
(NetOrk/deploy#6). So it is written once the pull has brought the images
onto the host, and before the first `up -d`. Positional rather than
behavioural, which is the honest limit of a static check; the behaviour is
in test_deploy_env_before_up.py.
"""
@staticmethod
@@ -118,14 +122,15 @@ class TestTheHostRecordsOnlyWhatItRuns:
assert text.count(needle) == 1, f"{needle!r} appears {text.count(needle)} times"
return text.index(needle)
def test_env_is_recorded_after_the_image_check(self) -> None:
verified = self._position("Verified: containers run")
recorded = self._position("Recording registry settings")
assert recorded > verified, (
"the host records NETORK_VERSION before the deploy is verified; "
"a failed deploy then leaves it pointing at a version never run"
)
def test_env_is_recorded_after_the_pull(self) -> None:
assert self._position("Recording registry settings") > self._position("Pulling images...")
def test_env_is_recorded_before_the_first_up(self) -> None:
lines = _logical_lines()
first_up = min(i for i, line in enumerate(lines) if "up -d" in line)
recorded = next(i for i, line in enumerate(lines) if "Recording registry settings" in line)
assert recorded < first_up, (
"containers are started before .env records NETORK_VERSION; "
"they would carry the previous deploy's version in their environment"
)
+91
View File
@@ -0,0 +1,91 @@
"""`--version` takes only the tag formats netOrk's CI publishes (NetOrk/netork#55, #208).
Every branch build is a `feature-` tag, with `/` turned into `-`: `feature/x` and
`feat/x` become `feature-x`, `fix/x` becomes `feature-fix-x`. The registry keeps
`feature-*` tags for 14 days and deletes a tag of any other form within a day, so
a bare `fix-x` is refused like any other unknown form. Anything else is refused
before a host is touched, so a typo cannot pull some ancient leftover tag.
The fake `ssh` only notes that it was reached and fails, which ends the deploy
right after the validation.
"""
from __future__ import annotations
import os
import subprocess
from pathlib import Path
import pytest
DEPLOY = Path(__file__).resolve().parent.parent / "deploy.sh"
def _deploy(tmp_path: Path, version: str) -> tuple[subprocess.CompletedProcess[str], bool]:
bin_dir = tmp_path / "bin"
bin_dir.mkdir()
marker = tmp_path / "ssh-was-called"
fake = bin_dir / "ssh"
fake.write_text(f"#!/bin/sh\ntouch {marker}\nexit 255\n")
fake.chmod(0o755)
result = subprocess.run(
["bash", str(DEPLOY), f"--version={version}", "testhost"],
env={
"PATH": f"{bin_dir}:{os.environ['PATH']}",
"HOME": str(tmp_path),
"DEPLOY_ENV_FILE": "/dev/null",
"REGISTRY_HOST": "registry.example",
},
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=30,
check=False,
)
return result, marker.exists()
@pytest.mark.parametrize(
"version",
[
"latest",
"latest-dev",
"main-1a2b3c4",
"feature-container-engine-api",
"feature-fix-a-hook-below-the-early-returns",
"feature-fix-ui-warning-icons",
"0.37.0",
],
)
def test_a_published_tag_is_accepted(tmp_path: Path, version: str) -> None:
result, reached_ssh = _deploy(tmp_path, version)
assert "Refusing to deploy version" not in result.stderr
assert reached_ssh
@pytest.mark.parametrize(
"version",
[
"bogus",
"fix-a-hook",
"fix/a-hook",
"feature-",
"feature-fix/ui/warning-icons",
"feature-fix-a;id",
"feature-$(id)",
"main-xyz1234",
],
)
def test_anything_else_is_refused_before_a_host_is_touched(tmp_path: Path, version: str) -> None:
result, reached_ssh = _deploy(tmp_path, version)
assert result.returncode != 0
assert f"Refusing to deploy version '{version}'" in result.stderr
assert not reached_ssh
def test_the_refusal_names_branch_builds(tmp_path: Path) -> None:
result, _ = _deploy(tmp_path, "bogus")
assert "feature-<branch> (also for fix/* branches)" in result.stderr