Files
deploy/deploy.sh
Christian Manivong 3a15c311a0
CI / check (pull_request) Successful in 19s
fix: record the version in .env before starting the containers
The services read ~/netork/.env through env_file, so whatever it says
when `up -d` creates a container ends up in that container's
environment. The settings were recorded only after the swap, so every
container carried the previous deploy's NETORK_VERSION: the image was
right, `docker exec <api> env` was not.

Record them once the pull has brought the images onto the host and
before the first `up -d`. A failed pull still leaves .env alone. If the
swap fails afterwards, .env names the tag whose compose files and
images are already on the host, which is what a hand-typed
`docker compose up -d` there should start.

Also say which branches a feature-<branch> tag comes from: feature/<name>
and feat/<name> become feature-<name>, and fix/<name> now becomes
feature-fix-<name> (NetOrk/netork#208), because the registry keeps only
feature-* branch tags. The accepted forms do not change; the comment, the
refusal message and the README now say so, and a test holds that
feature-fix-<name> passes and a bare fix-<name> does not.

Closes #6
2026-10-18 18:26:24 +02:00

507 lines
23 KiB
Bash

#!/usr/bin/env bash
# Deploy netOrk to one or more servers in parallel.
#
# Usage:
# ./deploy.sh [--version=<tag>] [server1 server2 ...]
#
# --version overrides the per-server NETORK_VERSION from deploy.env for this
# run only — useful to test one commit (--version=main-<sha>) without moving
# a server to a different release channel.
#
# Default targets and registry credentials are read from deploy.env next to
# this script (see deploy.env.example). DEPLOY_ENV_FILE points elsewhere.
#
# The servers pull pre-built images from REGISTRY_HOST. Nothing is built or
# copied from a working tree: the compose files come out of the engine image
# for the tag being deployed, and migrations run from that same image.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
ENV_FILE="${DEPLOY_ENV_FILE:-${HERE}/deploy.env}"
if [[ -f "$ENV_FILE" ]]; then
# shellcheck source=/dev/null
source "$ENV_FILE"
fi
DEPLOY_SERVERS="${DEPLOY_SERVERS:-}"
UI_SERVER="${UI_SERVER:-}"
REGISTRY_HOST="${REGISTRY_HOST:-}"
REGISTRY_USER="${REGISTRY_USER:-}"
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}"
NETORK_VERSION="${NETORK_VERSION:-latest}"
# The compose files the engine image carries under /app/deploy/.
COMPOSE_FILES="docker-compose.yml docker-compose.registry.yml"
# How long a deploy waits for another one on the same host before giving up.
DEPLOY_LOCK_WAIT="${DEPLOY_LOCK_WAIT:-900}"
read -ra SERVERS <<< "$DEPLOY_SERVERS"
# Run one step on a server, and let its failure stop the deploy.
#
# ssh reports the exit status of the *remote* command — and when that command
# ends in a pipe, the status is the last stage's, not the work's:
#
# ssh "$SERVER" "docker compose pull … 2>&1 | grep -E 'Pulled|Already|Error'"
#
# A failed pull prints a line containing `Error`, grep matches it and exits 0,
# and the deploy walks on to swap the containers. `set -o pipefail` cannot help:
# that pipeline ran in the remote shell, and from here the ssh succeeded.
#
# So the remote side stays pipe-free and the filtering happens here, where the
# status is still the one the command returned.
#
# run_remote <server> <what it is> <grep -E pattern, or "" for a tail> <command>
run_remote() {
local server=$1 label=$2 pattern=$3 command=$4
local output status
output="$(ssh -n "$server" "$command" 2>&1)" && status=0 || status=$?
if [[ $status -ne 0 ]]; then
printf '%s\n' "$output" | tail -25 | sed "s/^/[${server}] /" >&2
echo "[${server}] FAILED (exit ${status}): ${label}" >&2
return "$status"
fi
if [[ -n "$pattern" ]]; then
printf '%s\n' "$output" | grep -E "$pattern" | sed "s/^/[${server}] /" || true
else
printf '%s\n' "$output" | tail -6 | sed "s/^/[${server}] /"
fi
}
# ── The host's platform ──────────────────────────────────────────────────────
# From 0.34.0 on, netOrk's releases (X.Y.Z, latest) are published for
# linux/amd64 and linux/arm64, so a Raspberry Pi with a 64-bit OS runs them
# (NetOrk/netork#934). latest-dev, main-<sha>, feature-<branch> and every older
# release are amd64 only. On a Pi those fail at the first pull with "no
# matching manifest for linux/arm64/v8", which names neither the cause nor a
# tag that would work. So the host's architecture is compared with the
# platforms the registry lists for the tag, after the login (the registry may
# be private) and before anything on the host changes.
#
# Only a definite mismatch stops the deploy. When the registry says nothing
# readable (no `docker manifest`, a missing tag, a single-platform manifest),
# the pull decides, as before.
#
# check_platform <server> <version> <repositories>
check_platform() {
local server=$1 version=$2 repos=$3 answer arch missing
# shellcheck disable=SC2029 # registry, version and repositories expand here
answer="$(ssh -n "$server" "docker version --format 'arch={{.Server.Arch}}' 2>/dev/null; \
for r in ${repos}; do echo \"repo=\$r\"; \
docker manifest inspect '${REGISTRY_HOST}/netork/'\$r':${version}' 2>/dev/null; done; true")" || return 0
arch="$(sed -n 's/^arch=//p' <<< "$answer" | head -1)"
[[ -n "$arch" ]] || return 0
# One line per repository whose listed platforms lack this host's.
missing="$(awk -v want="$arch" '
/^repo=/ { repo = substr($0, 6); seen[repo] = ""; next }
repo != "" && match($0, /"architecture": *"[^"]+"/) {
a = substr($0, RSTART, RLENGTH); sub(/^"architecture": *"/, "", a); sub(/"$/, "", a)
if (a != "unknown") seen[repo] = seen[repo] " linux/" a
}
END {
for (r in seen)
if (seen[r] != "" && index(seen[r] " ", " linux/" want " ") == 0)
print r " has only" seen[r]
}' <<< "$answer")"
if [[ -n "$missing" ]]; then
echo "[${server}] Refusing to deploy '${version}': ${server} is linux/${arch}, and" >&2
while IFS= read -r line; do echo "[${server}] ${line}" >&2; done <<< "$missing"
echo "[${server}] netOrk publishes linux/arm64 for releases from 0.34.0 on (latest, X.Y.Z)." >&2
echo "[${server}] latest-dev, main-<sha> and feature-<branch> are linux/amd64 only." >&2
return 1
fi
}
# ── One deploy per host at a time ────────────────────────────────────────────
# Several sessions deploy to the same test server, and two runs used to overlap:
# on 2026-10-05 two `up -d --force-recreate` recreated each other's containers,
# on 2026-10-06 two runs renamed each other's *.new compose files, and with two
# tags one tag's compose files could start the other's images (NetOrk/deploy#1,
# #3). So each deploy first takes flock on ~/netork/.deploy.lock on the host.
#
# An ssh session holds it: the remote side opens the lock file on fd 9, takes
# the lock, says LOCKED, and then waits on its stdin. Closing that stdin ends
# the session and frees the lock, and so does anything else that ends it: the
# deploy failing, being interrupted, or losing the network. No stale lock file
# can outlive the deploy that took it.
#
# A host without flock (util-linux) is deployed without the lock, with a
# warning, rather than becoming undeployable.
#
# hold_host_lock <server> <who and what, for the next deploy to read>
hold_host_lock() {
local server=$1 info=$2 line
# shellcheck disable=SC2029 # the wait and the info are meant to expand here
coproc HOST_LOCK {
ssh "$server" "mkdir -p ~/netork && cd ~/netork && exec 9>.deploy.lock && \
if ! command -v flock >/dev/null 2>&1; then echo NOLOCK; exec cat >/dev/null; fi; \
if ! flock -n 9; then \
echo \"BUSY \$(cat .deploy.lock.info 2>/dev/null)\"; \
flock -w ${DEPLOY_LOCK_WAIT} 9 || { echo TIMEOUT; exit 1; }; \
fi; \
echo '${info}' > .deploy.lock.info; echo LOCKED; exec cat >/dev/null" 2>&1
}
while IFS= read -r -t "$(( DEPLOY_LOCK_WAIT + 60 ))" line <&"${HOST_LOCK[0]}"; do
case "$line" in
LOCKED) return 0 ;;
NOLOCK)
echo "[${server}] WARNING: no flock on ${server}; deploying without the host lock." >&2
return 0
;;
BUSY*)
echo "[${server}] another deploy holds ${server}: ${line#BUSY } — waiting up to ${DEPLOY_LOCK_WAIT}s..." >&2
;;
TIMEOUT)
echo "[${server}] the deploy lock on ${server} is still held after ${DEPLOY_LOCK_WAIT}s; giving up." >&2
return 1
;;
*) echo "[${server}] ${line}" >&2 ;;
esac
done
echo "[${server}] could not take the deploy lock on ${server} (the ssh session ended)." >&2
return 1
}
# Free the lock: closing the session's stdin ends the remote side.
release_host_lock() {
local fd="${HOST_LOCK[1]}" pid="${HOST_LOCK_PID}"
exec {fd}>&-
wait "$pid" 2>/dev/null || true
}
VERSION_OVERRIDE=""
EXPLICIT_SERVERS=()
i=1
while [[ $i -le $# ]]; do
arg="${!i}"
case "$arg" in
--version)
i=$(( i + 1 ))
VERSION_OVERRIDE="${!i:-}"
;;
--version=*) VERSION_OVERRIDE="${arg#--version=}" ;;
-*)
echo "Unknown option: ${arg}" >&2
exit 2
;;
*) EXPLICIT_SERVERS+=("$arg") ;;
esac
i=$(( i + 1 ))
done
if [[ ${#EXPLICIT_SERVERS[@]} -gt 0 ]]; then
SERVERS=("${EXPLICIT_SERVERS[@]}")
fi
# ── Version validation ───────────────────────────────────────────────────────
# A deploy only pulls images — it never verifies the pulled tag actually
# corresponds to a real, recently-built image for the commit/branch you think
# you're deploying. A stale or accidentally-reused tag (or a typo'd branch
# name that happens to collide with some ancient leftover tag) is pulled
# silently, with the only symptom surfacing later at `alembic upgrade head`,
# by which point containers are already switched over. Reject anything that
# doesn't match a tag format netOrk's CI actually publishes: latest,
# latest-dev, main-<sha>, feature-<branch>, or a release version (X.Y.Z).
# Every branch build is a feature-<branch> tag: feature/<name> and feat/<name>
# become feature-<name>, fix/<name> becomes feature-fix-<name>. The registry
# keeps feature-* tags for 14 days; a tag of any other form would be deleted
# within a day. See NetOrk/netork#55 and NetOrk/netork#208.
VALID_VERSION_RE='^(latest|latest-dev|main-[0-9a-f]{7,40}|feature-[A-Za-z0-9._-]+|[0-9]+\.[0-9]+\.[0-9]+)$'
_validate_version() {
local version="$1"
if [[ ! "$version" =~ $VALID_VERSION_RE ]]; then
echo "Refusing to deploy version '${version}': not a tag CI publishes (expected latest," \
"latest-dev, main-<sha>, feature-<branch> (also for fix/* branches), or X.Y.Z)." >&2
return 1
fi
}
if [[ -n "$VERSION_OVERRIDE" ]]; then
_validate_version "$VERSION_OVERRIDE"
fi
if [[ -z "$REGISTRY_HOST" ]]; then
echo "REGISTRY_HOST is not set (deploy.env or environment): nowhere to pull images from." >&2
exit 1
fi
if [[ ${#SERVERS[@]} -eq 0 ]]; then
echo "No servers given: pass them as arguments or set DEPLOY_SERVERS in deploy.env." >&2
exit 1
fi
# One server: under its host lock, the steps below.
#
# A step that fails stops the subshell (set -e) before release_host_lock runs.
# That is fine: the subshell's end closes the lock session's stdin all the same.
deploy_server() {
local SERVER="$1" KEY="${1//./_}" VER_VAR
VER_VAR="NETORK_VERSION_${KEY}"
hold_host_lock "$SERVER" \
"since $(date -u +%Y-%m-%dT%H:%M:%SZ), by ${USER:-?}@$(hostname -s), deploying ${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}"
deploy_steps "$SERVER"
release_host_lock
}
deploy_steps() {
local SERVER="$1"
# Per-server lookup — falls back to global values
local KEY="${SERVER//./_}"
local USER_VAR="REGISTRY_USER_${KEY}"
local PASS_VAR="REGISTRY_PASSWORD_${KEY}"
local VER_VAR="NETORK_VERSION_${KEY}"
local SERVER_USER="${!USER_VAR:-${REGISTRY_USER}}"
local SERVER_PASS="${!PASS_VAR:-${REGISTRY_PASSWORD}}"
local SERVER_VERSION="${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}"
_validate_version "$SERVER_VERSION" || return 1
local ENGINE_IMAGE="${REGISTRY_HOST}/netork/engine:${SERVER_VERSION}"
# The password goes over ssh's stdin. Inside the ssh argument it would be
# part of the remote shell's command line, readable through `ps` by every
# user on the host for as long as the login runs.
echo "[${SERVER}] Logging in to registry..."
# shellcheck disable=SC2029 # host and user are meant to expand here
printf '%s\n' "$SERVER_PASS" \
| ssh "$SERVER" "docker login '${REGISTRY_HOST}' -u '${SERVER_USER}' --password-stdin 2>&1" \
| sed "s/^/[${SERVER}] /"
local REPOS="engine"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
REPOS="engine ui"
fi
check_platform "$SERVER" "$SERVER_VERSION" "$REPOS"
# The compose files are versioned with the images they start, so they are
# taken from the engine image of this very tag rather than from whatever
# checkout happens to run the deploy. Both are written to .new first and only
# moved into place once both were read, so a failure leaves the previous pair
# intact. An image built before the files moved into it has no /app/deploy/.
echo "[${SERVER}] Fetching compose files from ${ENGINE_IMAGE}..."
run_remote "$SERVER" "fetching compose files from ${ENGINE_IMAGE}" 'Status|Error' \
"mkdir -p ~/netork && cd ~/netork && \
docker pull '${ENGINE_IMAGE}' && \
for f in ${COMPOSE_FILES}; do \
docker run --rm --entrypoint cat '${ENGINE_IMAGE}' /app/deploy/\$f > \$f.new \
|| { rm -f \$f.new; echo \"${ENGINE_IMAGE} carries no /app/deploy/\$f — the tag predates compose files in the engine image\"; exit 3; }; \
done && \
for f in ${COMPOSE_FILES}; do mv \$f.new \$f; done"
# Services running netOrk's own images. flower belongs here: it is the engine
# image with a different command, so leaving it out let the monitoring UI run
# nine days behind the workers it monitors, on an image that had since been
# untagged. See NetOrk/netork#95.
local SERVICES="netork-api netork-worker netork-poll-worker netork-ansible-worker netork-beat flower"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
SERVICES="${SERVICES} netork-ui"
fi
# Containers on upstream images. Reconciled, not force-recreated (below).
# postgres and redis are deliberately absent: restarting a database on every
# deploy would be worse than any compose change one could miss.
local INFRA_SERVICES="apt-cacher-ng signal-api"
echo "[${SERVER}] Pulling images..."
run_remote "$SERVER" "pulling images for ${SERVER_VERSION}" 'Pulled|Already|Error' \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}"
# docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project
# .env when they are not in the environment. This script passes both inline,
# but a hand-typed `docker compose up -d flower` on the host needs them there
# or dies on "invalid reference format". Values are rewritten in place, so
# the file does not grow a new line per deploy.
#
# Not earlier: this used to run before the pull, so a deploy that failed on a
# tag the registry does not have still left the host recording that tag, and
# the next `docker compose up` a human ran there reached for an image that
# does not exist. Once the pull went through, the images are on the host.
#
# Not later: the services also read this file (`env_file: .env`), so it is in
# every container's environment. Written after the swap, as it was for a
# while, it made each container carry the previous deploy's NETORK_VERSION
# (NetOrk/deploy#6). Should the swap below fail, the file names the tag whose
# compose files and images are already on the host, which is what a
# hand-typed `docker compose up -d` there should start.
echo "[${SERVER}] Recording registry settings in ~/netork/.env..."
# shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely
ssh "$SERVER" "bash -s" <<REMOTE_ENV
set -eu
cd ~/netork
touch .env
# Append to a file that does not end in a newline, and the last line grows a
# suffix instead of gaining a neighbour.
[ ! -s .env ] || [ -z "\$(tail -c1 .env)" ] || printf '\n' >> .env
for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do
key=\${kv%%=*}
if grep -q "^\${key}=" .env; then
sed -i "s|^\${key}=.*|\${kv}|" .env
else
printf '%s\n' "\${kv}" >> .env
fi
done
REMOTE_ENV
# --force-recreate is deliberate: `docker compose up -d` decides what to
# replace from the service definition's config hash, and the image reference
# (`.../ui:latest-dev`) does not change when the tag is moved to a new image.
# A deploy then leaves the old container running while reporting success —
# observed on netork-ui, which kept serving a stale image after its tag had
# already advanced. Recreating unconditionally costs a restart per deploy,
# which a deliberate rollout wants anyway. See NetOrk/netork#90.
#
# Tried twice. compose recreates the services in parallel, and on 2026-10-05 it
# lost a container it had just renamed ("No such container") and stopped with
# every engine container Created and none running: the old API was gone, so
# netOrk was down until someone ran the deploy again, which then went through
# (NetOrk/netork#586). The second attempt is that rerun. Should it fail as well,
# the container states go to the log and the deploy fails as before.
echo "[${SERVER}] Starting containers..."
local RECREATE="cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps --force-recreate ${SERVICES}"
if ! run_remote "$SERVER" "recreating ${SERVICES}" "" "$RECREATE"; then
local delay="${DEPLOY_RECREATE_RETRY_DELAY:-5}"
echo "[${SERVER}] Recreating failed; trying once more in ${delay}s..." >&2
sleep "$delay"
if ! run_remote "$SERVER" "recreating ${SERVICES} (second attempt)" "" "$RECREATE"; then
echo "[${SERVER}] Containers after two failed attempts:" >&2
ssh -n "$SERVER" "cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -a" 2>&1 \
| sed "s/^/[${SERVER}] /" >&2 || true
return 1
fi
fi
# Infrastructure containers, deliberately left out of the force-recreate
# above. Plain `up -d`: compose compares each service definition against the
# running container and recreates only what actually changed, so an edit to a
# healthcheck or an option in docker-compose.yml takes effect while a
# container nobody touched is left alone. signal-api is the reason this is
# not --force-recreate: it holds netOrk's Signal device link and recreating
# it every deploy is churn nobody asked for. See NetOrk/netork#95.
#
# `|| true`: infrastructure is reconciled opportunistically and its failure
# is not a reason to abandon a deploy that has already succeeded — but it
# prints as a failure instead of scrolling past.
echo "[${SERVER}] Reconciling infrastructure containers..."
run_remote "$SERVER" "reconciling ${INFRA_SERVICES}" "" \
"cd ~/netork && \
REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml \
up -d --no-deps ${INFRA_SERVICES}" || true
# Services netOrk no longer ships, with the data only they used. `docker
# compose up` never removes a container whose service has left the compose
# file, so each would keep running on every host until someone removed it by
# hand. Idempotent: a host that never had one, or was cleaned already, passes
# untouched. Containers first; a volume still in use cannot be removed.
# Only what exists is removed and reported: `docker rm -f` exits 0 for a
# container that is not there, so without the check every deploy said it had
# removed the registry, long after it was gone (NetOrk/deploy#7).
# registry (netork-registry-1, netork_registry_data): the bundled registry:2
# for satellite images. Satellites pull from registry.netork.io; it held one
# old image, nothing had pulled from it in 30 days, and it accepted
# anonymous pushes on port 5000 (NetOrk/netork#763).
local RETIRED_CONTAINERS="netork-registry-1"
local RETIRED_VOLUMES="netork_registry_data"
echo "[${SERVER}] Removing retired services..."
run_remote "$SERVER" "removing ${RETIRED_CONTAINERS} ${RETIRED_VOLUMES}" "" \
"for c in ${RETIRED_CONTAINERS}; do \
docker inspect \$c >/dev/null 2>&1 && docker rm -f \$c >/dev/null 2>&1 \
&& echo \"removed container \$c\"; done; \
for v in ${RETIRED_VOLUMES}; do \
docker volume rm \$v >/dev/null 2>&1 && echo \"removed volume \$v\"; done; true" || true
# Verify the containers actually run the image we just pulled. `docker
# compose up -d` reports "Running" (not "Started") when it decides nothing
# changed, and `pull` prints "Pulled" even when the tag was already local —
# so a deploy that had no effect at all is indistinguishable from a real one
# in the output above. Both the wrong-tag case (NETORK_VERSION pointing at a
# tag CI never republished) and a silently failed pull land here.
# See NetOrk/netork#88.
echo "[${SERVER}] Verifying containers run the pulled image..."
local VERIFY_PAIRS="netork-api:engine flower:engine"
if [[ " $UI_SERVER " == *" $SERVER "* ]]; then
VERIFY_PAIRS="${VERIFY_PAIRS} netork-ui:ui"
fi
local MISMATCH
MISMATCH=$(ssh -n "$SERVER" "cd ~/netork && for pair in ${VERIFY_PAIRS}; do \
svc=\${pair%%:*}; repo=\${pair##*:}; \
want=\$(docker image inspect --format '{{.Id}}' '${REGISTRY_HOST}/netork/'\${repo}':${SERVER_VERSION}' 2>/dev/null || true); \
cid=\$(REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \
docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -q \${svc} 2>/dev/null || true); \
got=\$(docker inspect --format '{{.Image}}' \${cid} 2>/dev/null || true); \
if [ -z \"\${want}\" ] || [ -z \"\${got}\" ] || [ \"\${want}\" != \"\${got}\" ]; then echo \${svc}; fi; \
done" || true)
if [[ -n "$MISMATCH" ]]; then
echo "[${SERVER}] ERROR: still running an older image after deploy: ${MISMATCH//$'\n'/ }" >&2
echo "[${SERVER}] Wanted tag '${SERVER_VERSION}'. On main pushes CI publishes" >&2
echo "[${SERVER}] 'latest-dev' and 'main-<sha>'; ':latest' only on version tags." >&2
return 1
fi
echo "[${SERVER}] Verified: containers run ${SERVER_VERSION}."
# netOrk migrates *after* the swap — its API tolerates the gap. What it does
# not tolerate is the migration failing unnoticed: the new code is already
# serving, so a missing column is a 500 on every request that touches it
# until someone looks. Loud, and non-zero. The revisions are the image's own.
echo "[${SERVER}] Running migrations..."
run_remote "$SERVER" "alembic upgrade head (containers are ALREADY swapped)" "" \
"cd ~/netork && docker compose exec -T netork-api alembic upgrade head"
echo "[${SERVER}] Pruning unused images..."
ssh -n "$SERVER" "docker image prune -af 2>&1 | tail -1" | sed "s/^/[${SERVER}] /"
echo "[${SERVER}] Done."
}
# ── Dispatch ──────────────────────────────────────────────────────────────────
echo "Deploying from ${REGISTRY_HOST} to: ${SERVERS[*]}"
export -f deploy_server deploy_steps hold_host_lock release_host_lock _validate_version run_remote \
check_platform
export UI_SERVER REGISTRY_HOST REGISTRY_USER REGISTRY_PASSWORD NETORK_VERSION VERSION_OVERRIDE \
VALID_VERSION_RE COMPOSE_FILES DEPLOY_LOCK_WAIT
# Export all per-server credential vars so subshells can resolve them
while IFS='=' read -r key _; do
if [[ "$key" =~ ^(REGISTRY_(USER|PASSWORD)|NETORK_VERSION)_ ]]; then
export "${key?}"
fi
done < <(compgen -v)
PIDS=()
for SERVER in "${SERVERS[@]}"; do
deploy_server "$SERVER" &
PIDS+=($!)
done
FAILED=()
for i in "${!PIDS[@]}"; do
if ! wait "${PIDS[$i]}"; then
FAILED+=("${SERVERS[$i]}")
fi
done
if [[ ${#FAILED[@]} -gt 0 ]]; then
echo "FAILED on: ${FAILED[*]}" >&2
exit 1
fi
echo "All servers deployed successfully."