Files
deploy/deploy.sh
T
Christian Manivong 30da942a7a
CI / check (pull_request) Successful in 18s
fix: hand ssl_certs to netork before the swap (NetOrk/netork#1389)
netork-ui ran nginx as root, and its entrypoint wrote the self-signed
certificate into ssl_certs as root, the key at 0600. From NetOrk/netork#1389
on, nginx runs as netork, the API's user, and cannot read that key, so the
UI would not start on any existing host.

Before the containers are recreated, one container of the engine image of
the version being deployed, as root with only CHOWN, runs
`chown -Rc netork:netork /app/ssl`. -c names only what it changed, so a
host handed over earlier sees nothing. A failure stops the deploy with the
old containers running. Older engine images carry the same netork user, so
deploying an older version stays possible.
2026-10-22 15:09:50 +02:00

570 lines
27 KiB
Bash

#!/usr/bin/env bash
# Deploy netOrk to one or more servers in parallel.
#
# Usage:
# ./deploy.sh [--version=<tag>] [--infra-db] [server1 server2 ...]
#
# --version overrides the per-server NETORK_VERSION from deploy.env for this
# run only — useful to test one commit (--version=main-<sha>) without moving
# a server to a different release channel.
#
# --infra-db lets the bundled PostgreSQL move to the volume the compose files
# name, by dump and restore (a new major version, NetOrk/netork#1308). Without
# it a deploy that would need that move stops before anything changed. Run it
# for one host at an announced time: the engine is down while the data moves.
#
# Default targets and registry credentials are read from deploy.env next to
# this script (see deploy.env.example). DEPLOY_ENV_FILE points elsewhere.
#
# The servers pull pre-built images from REGISTRY_HOST. Nothing is built or
# copied from a working tree: the compose files come out of the engine image
# for the tag being deployed, and migrations run from that same image.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
ENV_FILE="${DEPLOY_ENV_FILE:-${HERE}/deploy.env}"
if [[ -f "$ENV_FILE" ]]; then
# shellcheck source=/dev/null
source "$ENV_FILE"
fi
DEPLOY_SERVERS="${DEPLOY_SERVERS:-}"
UI_SERVER="${UI_SERVER:-}"
REGISTRY_HOST="${REGISTRY_HOST:-}"
REGISTRY_USER="${REGISTRY_USER:-}"
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
NETORK_VERSION="${NETORK_VERSION:-latest}"
# The compose files the engine image carries under /app/deploy/.
COMPOSE_FILES="docker-compose.yml docker-compose.registry.yml"
# How long a deploy waits for another one on the same host before giving up.
DEPLOY_LOCK_WAIT="${DEPLOY_LOCK_WAIT:-900}"
read -ra SERVERS <<< "$DEPLOY_SERVERS"
# Run one step on a server, and let its failure stop the deploy.
#
# ssh reports the exit status of the *remote* command — and when that command
# ends in a pipe, the status is the last stage's, not the work's:
#
# ssh "$SERVER" "docker compose pull … 2>&1 | grep -E 'Pulled|Already|Error'"
#
# A failed pull prints a line containing `Error`, grep matches it and exits 0,
# and the deploy walks on to swap the containers. `set -o pipefail` cannot help:
# that pipeline ran in the remote shell, and from here the ssh succeeded.
#
# So the remote side stays pipe-free and the filtering happens here, where the
# status is still the one the command returned.
#
# run_remote <server> <what it is> <grep -E pattern, or "" for a tail> <command>
run_remote() {
local server=$1 label=$2 pattern=$3 command=$4
local output status
output="$(ssh -n "$server" "$command" 2>&1)" && status=0 || status=$?
if [[ $status -ne 0 ]]; then
printf '%s\n' "$output" | tail -25 | sed "s/^/[${server}] /" >&2
echo "[${server}] FAILED (exit ${status}): ${label}" >&2
return "$status"
fi
if [[ -n "$pattern" ]]; then
printf '%s\n' "$output" | grep -E "$pattern" | sed "s/^/[${server}] /" || true
else
printf '%s\n' "$output" | tail -6 | sed "s/^/[${server}] /"
fi
}
# ── The host's platform ──────────────────────────────────────────────────────
# From 0.34.0 on, netOrk's releases (X.Y.Z, latest) are published for
# linux/amd64 and linux/arm64, so a Raspberry Pi with a 64-bit OS runs them
# (NetOrk/netork#934). latest-dev, main-<sha>, feature-<branch> and every older
# release are amd64 only. On a Pi those fail at the first pull with "no
# matching manifest for linux/arm64/v8", which names neither the cause nor a
# tag that would work. So the host's architecture is compared with the
# platforms the registry lists for the tag, after the login (the registry may
# be private) and before anything on the host changes.
#
# Only a definite mismatch stops the deploy. When the registry says nothing
# readable (no `docker manifest`, a missing tag, a single-platform manifest),
# the pull decides, as before.
#
# check_platform <server> <version> <repositories>
check_platform() {
local server=$1 version=$2 repos=$3 answer arch missing
# shellcheck disable=SC2029 # registry, version and repositories expand here
answer="$(ssh -n "$server" "docker version --format 'arch={{.Server.Arch}}' 2>/dev/null; \
for r in ${repos}; do echo \"repo=\$r\"; \
docker manifest inspect '${REGISTRY_HOST}/netork/'\$r':${version}' 2>/dev/null; done; true")" || return 0
arch="$(sed -n 's/^arch=//p' <<< "$answer" | head -1)"
[[ -n "$arch" ]] || return 0
# One line per repository whose listed platforms lack this host's.
missing="$(awk -v want="$arch" '
/^repo=/ { repo = substr($0, 6); seen[repo] = ""; next }
repo != "" && match($0, /"architecture": *"[^"]+"/) {
a = substr($0, RSTART, RLENGTH); sub(/^"architecture": *"/, "", a); sub(/"$/, "", a)
if (a != "unknown") seen[repo] = seen[repo] " linux/" a
}
END {
for (r in seen)
if (seen[r] != "" && index(seen[r] " ", " linux/" want " ") == 0)
print r " has only" seen[r]
}' <<< "$answer")"
if [[ -n "$missing" ]]; then
echo "[${server}] Refusing to deploy '${version}': ${server} is linux/${arch}, and" >&2
while IFS= read -r line; do echo "[${server}] ${line}" >&2; done <<< "$missing"
echo "[${server}] netOrk publishes linux/arm64 for releases from 0.34.0 on (latest, X.Y.Z)." >&2
echo "[${server}] latest-dev, main-<sha> and feature-<branch> are linux/amd64 only." >&2
return 1
fi
}
# ── One deploy per host at a time ────────────────────────────────────────────
# Several sessions deploy to the same test server, and two runs used to overlap:
# on 2026-10-05 two `up -d --force-recreate` recreated each other's containers,
# on 2026-10-06 two runs renamed each other's *.new compose files, and with two
# tags one tag's compose files could start the other's images (NetOrk/deploy#1,
# #3). So each deploy first takes flock on ~/netork/.deploy.lock on the host.
#
# An ssh session holds it: the remote side opens the lock file on fd 9, takes
# the lock, says LOCKED, and then waits on its stdin. Closing that stdin ends
# the session and frees the lock, and so does anything else that ends it: the
# deploy failing, being interrupted, or losing the network. No stale lock file
# can outlive the deploy that took it.
#
# A host without flock (util-linux) is deployed without the lock, with a
# warning, rather than becoming undeployable.
#
# hold_host_lock <server> <who and what, for the next deploy to read>
hold_host_lock() {
local server=$1 info=$2 line
# shellcheck disable=SC2029 # the wait and the info are meant to expand here
coproc HOST_LOCK {
ssh "$server" "mkdir -p ~/netork && cd ~/netork && exec 9>.deploy.lock && \
if ! command -v flock >/dev/null 2>&1; then echo NOLOCK; exec cat >/dev/null; fi; \
if ! flock -n 9; then \
echo \"BUSY \$(cat .deploy.lock.info 2>/dev/null)\"; \
flock -w ${DEPLOY_LOCK_WAIT} 9 || { echo TIMEOUT; exit 1; }; \
fi; \
echo '${info}' > .deploy.lock.info; echo LOCKED; exec cat >/dev/null" 2>&1
}
while IFS= read -r -t "$(( DEPLOY_LOCK_WAIT + 60 ))" line <&"${HOST_LOCK[0]}"; do
case "$line" in
LOCKED) return 0 ;;
NOLOCK)
echo "[${server}] WARNING: no flock on ${server}; deploying without the host lock." >&2
return 0
;;
BUSY*)
echo "[${server}] another deploy holds ${server}: ${line#BUSY } — waiting up to ${DEPLOY_LOCK_WAIT}s..." >&2
;;
TIMEOUT)
echo "[${server}] the deploy lock on ${server} is still held after ${DEPLOY_LOCK_WAIT}s; giving up." >&2
return 1
;;
*) echo "[${server}] ${line}" >&2 ;;
esac
done
echo "[${server}] could not take the deploy lock on ${server} (the ssh session ended)." >&2
return 1
}
# Free the lock: closing the session's stdin ends the remote side.
release_host_lock() {
local fd="${HOST_LOCK[1]}" pid="${HOST_LOCK_PID}"
exec {fd}>&-
wait "$pid" 2>/dev/null || true
}
VERSION_OVERRIDE=""
DATABASE_MODE=reconcile
EXPLICIT_SERVERS=()
i=1
while [[ $i -le $# ]]; do
arg="${!i}"
case "$arg" in
--version)
i=$(( i + 1 ))
VERSION_OVERRIDE="${!i:-}"
;;
--version=*) VERSION_OVERRIDE="${arg#--version=}" ;;
--infra-db) DATABASE_MODE=move ;;
-*)
echo "Unknown option: ${arg}" >&2
exit 2
;;
*) EXPLICIT_SERVERS+=("$arg") ;;
esac
i=$(( i + 1 ))
done
if [[ ${#EXPLICIT_SERVERS[@]} -gt 0 ]]; then
SERVERS=("${EXPLICIT_SERVERS[@]}")
fi
# ── Version validation ───────────────────────────────────────────────────────
# A deploy only pulls images — it never verifies the pulled tag actually
# corresponds to a real, recently-built image for the commit/branch you think
# you're deploying. A stale or accidentally-reused tag (or a typo'd branch
# name that happens to collide with some ancient leftover tag) is pulled
# silently, with the only symptom surfacing later at `alembic upgrade head`,
# by which point containers are already switched over. Reject anything that
# doesn't match a tag format netOrk's CI actually publishes: latest,
# latest-dev, main-<sha>, feature-<branch>, or a release version (X.Y.Z).
# Every branch build is a feature-<branch> tag: feature/<name> and feat/<name>
# become feature-<name>, fix/<name> becomes feature-fix-<name>. The registry
# keeps feature-* tags for 14 days; a tag of any other form would be deleted
# within a day. See NetOrk/netork#55 and NetOrk/netork#208.
VALID_VERSION_RE='^(latest|latest-dev|main-[0-9a-f]{7,40}|feature-[A-Za-z0-9._-]+|[0-9]+\.[0-9]+\.[0-9]+)$'
_validate_version() {
local version="$1"
if [[ ! "$version" =~ $VALID_VERSION_RE ]]; then
echo "Refusing to deploy version '${version}': not a tag CI publishes (expected latest," \
"latest-dev, main-<sha>, feature-<branch> (also for fix/* branches), or X.Y.Z)." >&2
return 1
fi
}
if [[ -n "$VERSION_OVERRIDE" ]]; then
_validate_version "$VERSION_OVERRIDE"
fi
if [[ -z "$REGISTRY_HOST" ]]; then
echo "REGISTRY_HOST is not set (deploy.env or environment): nowhere to pull images from." >&2
exit 1
fi
if [[ ${#SERVERS[@]} -eq 0 ]]; then
echo "No servers given: pass them as arguments or set DEPLOY_SERVERS in deploy.env." >&2
exit 1
fi
# One server: under its host lock, the steps below.
#
# A step that fails stops the subshell (set -e) before release_host_lock runs.
# That is fine: the subshell's end closes the lock session's stdin all the same.
deploy_server() {
local SERVER="$1" KEY="${1//./_}" VER_VAR
VER_VAR="NETORK_VERSION_${KEY}"
hold_host_lock "$SERVER" \
"since $(date -u +%Y-%m-%dT%H:%M:%SZ), by ${USER:-?}@$(hostname -s), deploying ${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}"
deploy_steps "$SERVER"
release_host_lock
}
deploy_steps() {
local SERVER="$1"
# Per-server lookup — falls back to global values
local KEY="${SERVER//./_}"
local USER_VAR="REGISTRY_USER_${KEY}"
local PASS_VAR="REGISTRY_PASSWORD_${KEY}"
local VER_VAR="NETORK_VERSION_${KEY}"
local SERVER_USER="${!USER_VAR:-${REGISTRY_USER}}"
local SERVER_PASS="${!PASS_VAR:-${REGISTRY_PASSWORD}}"
local SERVER_VERSION="${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}"
_validate_version "$SERVER_VERSION" || return 1
local ENGINE_IMAGE="${REGISTRY_HOST}/netork/engine:${SERVER_VERSION}"
# The password goes over ssh's stdin. Inside the ssh argument it would be
# part of the remote shell's command line, readable through `ps` by every
# user on the host for as long as the login runs.
echo "[${SERVER}] Logging in to registry..."
# shellcheck disable=SC2029 # host and user are meant to expand here
printf '%s\n' "$SERVER_PASS" \
| ssh "$SERVER" "docker login '${REGISTRY_HOST}' -u '${SERVER_USER}' --password-stdin 2>&1" \
| sed "s/^/[${SERVER}] /"
local REPOS="engine"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
REPOS="engine ui"
fi
check_platform "$SERVER" "$SERVER_VERSION" "$REPOS"
# The compose files are versioned with the images they start, so they are
# taken from the engine image of this very tag rather than from whatever
# checkout happens to run the deploy. Both are written to .new first and only
# moved into place once both were read, so a failure leaves the previous pair
# intact. An image built before the files moved into it has no /app/deploy/.
echo "[${SERVER}] Fetching compose files from ${ENGINE_IMAGE}..."
run_remote "$SERVER" "fetching compose files from ${ENGINE_IMAGE}" 'Status|Error' \
"mkdir -p ~/netork && cd ~/netork && \
docker pull '${ENGINE_IMAGE}' && \
for f in ${COMPOSE_FILES}; do \
docker run --rm --entrypoint cat '${ENGINE_IMAGE}' /app/deploy/\$f > \$f.new \
|| { rm -f \$f.new; echo \"${ENGINE_IMAGE} carries no /app/deploy/\$f — the tag predates compose files in the engine image\"; exit 3; }; \
done && \
for f in ${COMPOSE_FILES}; do \
if [ -f \"\$f\" ]; then cp \"\$f\" \"\$f.prev\"; fi; mv \$f.new \$f; \
done"
# Services running netOrk's own images. flower belongs here: it is the engine
# image with a different command, so leaving it out let the monitoring UI run
# nine days behind the workers it monitors, on an image that had since been
# untagged. See NetOrk/netork#95.
local SERVICES="netork-api netork-worker netork-poll-worker netork-ansible-worker netork-beat flower"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
SERVICES="${SERVICES} netork-ui"
fi
# Containers on upstream images. Reconciled, not force-recreated (below).
# postgres is deliberately absent: restarting a database on every deploy would
# be worse than any compose change one could miss. redis is reconciled on its
# own, before the engine (below).
local INFRA_SERVICES="apt-cacher-ng signal-api"
echo "[${SERVER}] Pulling images..."
run_remote "$SERVER" "pulling images for ${SERVER_VERSION}" 'Pulled|Already|Error' \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}"
# The bundled database, before .env, broker and engine (remote/database.sh). A
# changed image on the same volume (a minor release) is recreated; a new
# volume (a major release) moves only with --infra-db, and without it the
# deploy stops here, nothing changed and the old engine running. The script
# goes over stdin, so the host needs nothing but bash and docker.
echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..."
local db_status=0
# shellcheck disable=SC2029 # mode, services and version expand here on purpose
ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \
< "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$?
if [[ $db_status -eq 3 ]]; then
# Refused, nothing changed: the host keeps the compose files it had, so a
# hand-typed `docker compose up -d` cannot carry the database to the other
# volume. .env is written only after this step.
ssh -n "$SERVER" "cd ~/netork && for f in ${COMPOSE_FILES}; do if [ -f \"\$f.prev\" ]; then mv \"\$f.prev\" \"\$f\"; fi; done" || true
echo "[${SERVER}] FAILED: the database step refused; the host is as it was" >&2
return 1
fi
if [[ $db_status -ne 0 ]]; then
echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2
return 1
fi
# docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project
# .env when they are not in the environment. This script passes both inline,
# but a hand-typed `docker compose up -d flower` on the host needs them there
# or dies on "invalid reference format". Values are rewritten in place, so
# the file does not grow a new line per deploy.
#
# Not earlier: this used to run before the pull, so a deploy that failed on a
# tag the registry does not have still left the host recording that tag, and
# the next `docker compose up` a human ran there reached for an image that
# does not exist. Once the pull went through, the images are on the host.
#
# Not later: the services also read this file (`env_file: .env`), so it is in
# every container's environment. Written after the swap, as it was for a
# while, it made each container carry the previous deploy's NETORK_VERSION
# (NetOrk/deploy#6). Should the swap below fail, the file names the tag whose
# compose files and images are already on the host, which is what a
# hand-typed `docker compose up -d` there should start.
echo "[${SERVER}] Recording registry settings in ~/netork/.env..."
# shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely
ssh "$SERVER" "bash -s" <<REMOTE_ENV
set -eu
cd ~/netork
touch .env
# Append to a file that does not end in a newline, and the last line grows a
# suffix instead of gaining a neighbour.
[ ! -s .env ] || [ -z "\$(tail -c1 .env)" ] || printf '\n' >> .env
for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do
key=\${kv%%=*}
if grep -q "^\${key}=" .env; then
sed -i "s|^\${key}=.*|\${kv}|" .env
else
printf '%s\n' "\${kv}" >> .env
fi
done
REMOTE_ENV
# The broker, before the engine that cannot work without it. Plain `up -d`
# recreates it only when its compose definition changed, so an unchanged
# broker keeps running and keeps its queue. It used to be in no list at all,
# and a changed definition -- Valkey in place of redis:7-alpine
# (NetOrk/netork#1308) -- never reached a host. --wait: the engine comes up
# against a healthy broker. If it does not get there, the deploy stops here,
# with the old engine still running.
echo "[${SERVER}] Reconciling the broker..."
run_remote "$SERVER" "reconciling redis" "" \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps --wait --wait-timeout 120 redis"
# ssl_certs to netork, before the swap (NetOrk/netork#1388, #1389). netork-ui
# ran nginx as root, and its entrypoint wrote the self-signed certificate into
# this volume as root, the key at 0600. nginx now runs as netork, the API's
# user, and cannot read that key: the UI would not start. One container of the
# engine image of this version, as root with only the capability to chown,
# hands the volume over. `chown -c` names only what it changed, so once a host
# is handed over this does nothing and says nothing. If it fails, the deploy
# stops here, with the old containers running.
echo "[${SERVER}] Handing ssl_certs to netork..."
run_remote "$SERVER" "handing ssl_certs to netork" "" \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api \
-Rc netork:netork /app/ssl"
# --force-recreate is deliberate: `docker compose up -d` decides what to
# replace from the service definition's config hash, and the image reference
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
# A deploy then leaves the old container running while reporting success —
# observed on netork-ui, which kept serving a stale image after its tag had
# already advanced. Recreating unconditionally costs a restart per deploy,
# which a deliberate rollout wants anyway. See NetOrk/netork#90.
#
# Tried twice. compose recreates the services in parallel, and on 2026-10-05 it
# lost a container it had just renamed ("No such container") and stopped with
# every engine container Created and none running: the old API was gone, so
# netOrk was down until someone ran the deploy again, which then went through
# (NetOrk/netork#586). The second attempt is that rerun. Should it fail as well,
# the container states go to the log and the deploy fails as before.
echo "[${SERVER}] Starting containers..."
local RECREATE="cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps --force-recreate ${SERVICES}"
if ! run_remote "$SERVER" "recreating ${SERVICES}" "" "$RECREATE"; then
local delay="${DEPLOY_RECREATE_RETRY_DELAY:-5}"
echo "[${SERVER}] Recreating failed; trying once more in ${delay}s..." >&2
sleep "$delay"
if ! run_remote "$SERVER" "recreating ${SERVICES} (second attempt)" "" "$RECREATE"; then
echo "[${SERVER}] Containers after two failed attempts:" >&2
ssh -n "$SERVER" "cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -a" 2>&1 \
| sed "s/^/[${SERVER}] /" >&2 || true
return 1
fi
fi
# Infrastructure containers, deliberately left out of the force-recreate
# above. Plain `up -d`: compose compares each service definition against the
# running container and recreates only what actually changed, so an edit to a
# healthcheck or an option in docker-compose.yml takes effect while a
# container nobody touched is left alone. signal-api is the reason this is
# not --force-recreate: it holds netOrk's Signal device link and recreating
# it every deploy is churn nobody asked for. See NetOrk/netork#95.
#
# `|| true`: infrastructure is reconciled opportunistically and its failure
# is not a reason to abandon a deploy that has already succeeded — but it
# prints as a failure instead of scrolling past.
echo "[${SERVER}] Reconciling infrastructure containers..."
run_remote "$SERVER" "reconciling ${INFRA_SERVICES}" "" \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps ${INFRA_SERVICES}" || true
# Services netOrk no longer ships, with the data only they used. `docker
# compose up` never removes a container whose service has left the compose
# file, so each would keep running on every host until someone removed it by
# hand. Idempotent: a host that never had one, or was cleaned already, passes
# untouched. Containers first; a volume still in use cannot be removed.
# Only what exists is removed and reported: `docker rm -f` exits 0 for a
# container that is not there, so without the check every deploy said it had
# removed the registry, long after it was gone (NetOrk/deploy#7).
# registry (netork-registry-1, netork_registry_data): the bundled registry:2
# for satellite images. Satellites pull from registry.netork.io; it held one
# old image, nothing had pulled from it in 30 days, and it accepted
# anonymous pushes on port 5000 (NetOrk/netork#763).
local RETIRED_CONTAINERS="netork-registry-1"
local RETIRED_VOLUMES="netork_registry_data"
echo "[${SERVER}] Removing retired services..."
run_remote "$SERVER" "removing ${RETIRED_CONTAINERS} ${RETIRED_VOLUMES}" "" \
"for c in ${RETIRED_CONTAINERS}; do \
docker inspect \$c >/dev/null 2>&1 && docker rm -f \$c >/dev/null 2>&1 \
&& echo \"removed container \$c\"; done; \
for v in ${RETIRED_VOLUMES}; do \
docker volume rm \$v >/dev/null 2>&1 && echo \"removed volume \$v\"; done; true" || true
# Verify the containers actually run the image we just pulled. `docker
# compose up -d` reports "Running" (not "Started") when it decides nothing
# changed, and `pull` prints "Pulled" even when the tag was already local —
# so a deploy that had no effect at all is indistinguishable from a real one
# in the output above. Both the wrong-tag case (NETORK_VERSION pointing at a
# tag CI never republished) and a silently failed pull land here.
# See NetOrk/netork#88.
echo "[${SERVER}] Verifying containers run the pulled image..."
local VERIFY_PAIRS="netork-api:engine flower:engine"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
VERIFY_PAIRS="${VERIFY_PAIRS} netork-ui:ui"
fi
local MISMATCH
MISMATCH=$(ssh -n "$SERVER" "cd ~/netork && for pair in ${VERIFY_PAIRS}; do \
svc=\${pair%%:*}; repo=\${pair##*:}; \
want=\$(docker image inspect --format '{{.Id}}' '${REGISTRY_HOST}/netork/'\${repo}':${SERVER_VERSION}' 2>/dev/null || true); \
cid=\$(REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -q \${svc} 2>/dev/null || true); \
got=\$(docker inspect --format '{{.Image}}' \${cid} 2>/dev/null || true); \
if [ -z \"\${want}\" ] || [ -z \"\${got}\" ] || [ \"\${want}\" != \"\${got}\" ]; then echo \${svc}; fi; \
done" || true)
if [[ -n "$MISMATCH" ]]; then
echo "[${SERVER}] ERROR: still running an older image after deploy: ${MISMATCH//$'\n'/ }" >&2
echo "[${SERVER}] Wanted tag '${SERVER_VERSION}'. On main pushes CI publishes" >&2
echo "[${SERVER}] 'latest-dev' and 'main-<sha>'; ':latest' only on version tags." >&2
return 1
fi
echo "[${SERVER}] Verified: containers run ${SERVER_VERSION}."
# netOrk migrates *after* the swap — its API tolerates the gap. What it does
# not tolerate is the migration failing unnoticed: the new code is already
# serving, so a missing column is a 500 on every request that touches it
# until someone looks. Loud, and non-zero. The revisions are the image's own.
echo "[${SERVER}] Running migrations..."
run_remote "$SERVER" "alembic upgrade head (containers are ALREADY swapped)" "" \
"cd ~/netork && docker compose exec -T netork-api alembic upgrade head"
echo "[${SERVER}] Pruning unused images..."
ssh -n "$SERVER" "docker image prune -af 2>&1 | tail -1" | sed "s/^/[${SERVER}] /"
echo "[${SERVER}] Done."
}
# ── Dispatch ──────────────────────────────────────────────────────────────────
echo "Deploying from ${REGISTRY_HOST} to: ${SERVERS[*]}"
export -f deploy_server deploy_steps hold_host_lock release_host_lock _validate_version run_remote \
check_platform
export UI_SERVER REGISTRY_HOST REGISTRY_USER REGISTRY_PASSWORD NETORK_VERSION VERSION_OVERRIDE \
VALID_VERSION_RE COMPOSE_FILES DEPLOY_LOCK_WAIT
# Export all per-server credential vars so subshells can resolve them
while IFS='=' read -r key _; do
if [[ "$key" =~ ^(REGISTRY_(USER|PASSWORD)|NETORK_VERSION)_ ]]; then
export "${key?}"
fi
done < <(compgen -v)
PIDS=()
for SERVER in "${SERVERS[@]}"; do
deploy_server "$SERVER" &
PIDS+=($!)
done
FAILED=()
for i in "${!PIDS[@]}"; do
if ! wait "${PIDS[$i]}"; then
FAILED+=("${SERVERS[$i]}")
fi
done
if [[ ${#FAILED[@]} -gt 0 ]]; then
echo "FAILED on: ${FAILED[*]}" >&2
exit 1
fi
echo "All servers deployed successfully."