#!/usr/bin/env bash # Deploy netOrk to one or more servers in parallel. # # Usage: # ./deploy.sh [--version=] [--infra-db] [server1 server2 ...] # # --version overrides the per-server NETORK_VERSION from deploy.env for this # run only — useful to test one commit (--version=main-) without moving # a server to a different release channel. # # --infra-db lets the bundled PostgreSQL move to the volume the compose files # name, by dump and restore (a new major version, NetOrk/netork#1308). Without # it a deploy that would need that move stops before anything changed. Run it # for one host at an announced time: the engine is down while the data moves. # # Default targets and registry credentials are read from deploy.env next to # this script (see deploy.env.example). DEPLOY_ENV_FILE points elsewhere. # # The servers pull pre-built images from REGISTRY_HOST. Nothing is built or # copied from a working tree: the compose files come out of the engine image # for the tag being deployed, and migrations run from that same image. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" ENV_FILE="${DEPLOY_ENV_FILE:-${HERE}/deploy.env}" if [[ -f "$ENV_FILE" ]]; then # shellcheck source=/dev/null source "$ENV_FILE" fi DEPLOY_SERVERS="${DEPLOY_SERVERS:-}" UI_SERVER="${UI_SERVER:-}" REGISTRY_HOST="${REGISTRY_HOST:-}" REGISTRY_USER="${REGISTRY_USER:-}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-}" NETORK_VERSION="${NETORK_VERSION:-latest}" # The compose files the engine image carries under /app/deploy/. COMPOSE_FILES="docker-compose.yml docker-compose.registry.yml" # How long a deploy waits for another one on the same host before giving up. DEPLOY_LOCK_WAIT="${DEPLOY_LOCK_WAIT:-900}" read -ra SERVERS <<< "$DEPLOY_SERVERS" # Run one step on a server, and let its failure stop the deploy. # # ssh reports the exit status of the *remote* command — and when that command # ends in a pipe, the status is the last stage's, not the work's: # # ssh "$SERVER" "docker compose pull … 2>&1 | grep -E 'Pulled|Already|Error'" # # A failed pull prints a line containing `Error`, grep matches it and exits 0, # and the deploy walks on to swap the containers. `set -o pipefail` cannot help: # that pipeline ran in the remote shell, and from here the ssh succeeded. # # So the remote side stays pipe-free and the filtering happens here, where the # status is still the one the command returned. # # run_remote run_remote() { local server=$1 label=$2 pattern=$3 command=$4 local output status output="$(ssh -n "$server" "$command" 2>&1)" && status=0 || status=$? if [[ $status -ne 0 ]]; then printf '%s\n' "$output" | tail -25 | sed "s/^/[${server}] /" >&2 echo "[${server}] FAILED (exit ${status}): ${label}" >&2 return "$status" fi if [[ -n "$pattern" ]]; then printf '%s\n' "$output" | grep -E "$pattern" | sed "s/^/[${server}] /" || true else printf '%s\n' "$output" | tail -6 | sed "s/^/[${server}] /" fi } # ── The host's platform ────────────────────────────────────────────────────── # From 0.34.0 on, netOrk's releases (X.Y.Z, latest) are published for # linux/amd64 and linux/arm64, so a Raspberry Pi with a 64-bit OS runs them # (NetOrk/netork#934). latest-dev, main-, feature- and every older # release are amd64 only. On a Pi those fail at the first pull with "no # matching manifest for linux/arm64/v8", which names neither the cause nor a # tag that would work. So the host's architecture is compared with the # platforms the registry lists for the tag, after the login (the registry may # be private) and before anything on the host changes. # # Only a definite mismatch stops the deploy. When the registry says nothing # readable (no `docker manifest`, a missing tag, a single-platform manifest), # the pull decides, as before. # # check_platform check_platform() { local server=$1 version=$2 repos=$3 answer arch missing # shellcheck disable=SC2029 # registry, version and repositories expand here answer="$(ssh -n "$server" "docker version --format 'arch={{.Server.Arch}}' 2>/dev/null; \ for r in ${repos}; do echo \"repo=\$r\"; \ docker manifest inspect '${REGISTRY_HOST}/netork/'\$r':${version}' 2>/dev/null; done; true")" || return 0 arch="$(sed -n 's/^arch=//p' <<< "$answer" | head -1)" [[ -n "$arch" ]] || return 0 # One line per repository whose listed platforms lack this host's. missing="$(awk -v want="$arch" ' /^repo=/ { repo = substr($0, 6); seen[repo] = ""; next } repo != "" && match($0, /"architecture": *"[^"]+"/) { a = substr($0, RSTART, RLENGTH); sub(/^"architecture": *"/, "", a); sub(/"$/, "", a) if (a != "unknown") seen[repo] = seen[repo] " linux/" a } END { for (r in seen) if (seen[r] != "" && index(seen[r] " ", " linux/" want " ") == 0) print r " has only" seen[r] }' <<< "$answer")" if [[ -n "$missing" ]]; then echo "[${server}] Refusing to deploy '${version}': ${server} is linux/${arch}, and" >&2 while IFS= read -r line; do echo "[${server}] ${line}" >&2; done <<< "$missing" echo "[${server}] netOrk publishes linux/arm64 for releases from 0.34.0 on (latest, X.Y.Z)." >&2 echo "[${server}] latest-dev, main- and feature- are linux/amd64 only." >&2 return 1 fi } # ── One deploy per host at a time ──────────────────────────────────────────── # Several sessions deploy to the same test server, and two runs used to overlap: # on 2026-10-05 two `up -d --force-recreate` recreated each other's containers, # on 2026-10-06 two runs renamed each other's *.new compose files, and with two # tags one tag's compose files could start the other's images (NetOrk/deploy#1, # #3). So each deploy first takes flock on ~/netork/.deploy.lock on the host. # # An ssh session holds it: the remote side opens the lock file on fd 9, takes # the lock, says LOCKED, and then waits on its stdin. Closing that stdin ends # the session and frees the lock, and so does anything else that ends it: the # deploy failing, being interrupted, or losing the network. No stale lock file # can outlive the deploy that took it. # # A host without flock (util-linux) is deployed without the lock, with a # warning, rather than becoming undeployable. # # hold_host_lock hold_host_lock() { local server=$1 info=$2 line # shellcheck disable=SC2029 # the wait and the info are meant to expand here coproc HOST_LOCK { ssh "$server" "mkdir -p ~/netork && cd ~/netork && exec 9>.deploy.lock && \ if ! command -v flock >/dev/null 2>&1; then echo NOLOCK; exec cat >/dev/null; fi; \ if ! flock -n 9; then \ echo \"BUSY \$(cat .deploy.lock.info 2>/dev/null)\"; \ flock -w ${DEPLOY_LOCK_WAIT} 9 || { echo TIMEOUT; exit 1; }; \ fi; \ echo '${info}' > .deploy.lock.info; echo LOCKED; exec cat >/dev/null" 2>&1 } while IFS= read -r -t "$(( DEPLOY_LOCK_WAIT + 60 ))" line <&"${HOST_LOCK[0]}"; do case "$line" in LOCKED) return 0 ;; NOLOCK) echo "[${server}] WARNING: no flock on ${server}; deploying without the host lock." >&2 return 0 ;; BUSY*) echo "[${server}] another deploy holds ${server}: ${line#BUSY } — waiting up to ${DEPLOY_LOCK_WAIT}s..." >&2 ;; TIMEOUT) echo "[${server}] the deploy lock on ${server} is still held after ${DEPLOY_LOCK_WAIT}s; giving up." >&2 return 1 ;; *) echo "[${server}] ${line}" >&2 ;; esac done echo "[${server}] could not take the deploy lock on ${server} (the ssh session ended)." >&2 return 1 } # Free the lock: closing the session's stdin ends the remote side. release_host_lock() { local fd="${HOST_LOCK[1]}" pid="${HOST_LOCK_PID}" exec {fd}>&- wait "$pid" 2>/dev/null || true } VERSION_OVERRIDE="" DATABASE_MODE=reconcile EXPLICIT_SERVERS=() i=1 while [[ $i -le $# ]]; do arg="${!i}" case "$arg" in --version) i=$(( i + 1 )) VERSION_OVERRIDE="${!i:-}" ;; --version=*) VERSION_OVERRIDE="${arg#--version=}" ;; --infra-db) DATABASE_MODE=move ;; -*) echo "Unknown option: ${arg}" >&2 exit 2 ;; *) EXPLICIT_SERVERS+=("$arg") ;; esac i=$(( i + 1 )) done if [[ ${#EXPLICIT_SERVERS[@]} -gt 0 ]]; then SERVERS=("${EXPLICIT_SERVERS[@]}") fi # ── Version validation ─────────────────────────────────────────────────────── # A deploy only pulls images — it never verifies the pulled tag actually # corresponds to a real, recently-built image for the commit/branch you think # you're deploying. A stale or accidentally-reused tag (or a typo'd branch # name that happens to collide with some ancient leftover tag) is pulled # silently, with the only symptom surfacing later at `alembic upgrade head`, # by which point containers are already switched over. Reject anything that # doesn't match a tag format netOrk's CI actually publishes: latest, # latest-dev, main-, feature-, or a release version (X.Y.Z). # Every branch build is a feature- tag: feature/ and feat/ # become feature-, fix/ becomes feature-fix-. The registry # keeps feature-* tags for 14 days; a tag of any other form would be deleted # within a day. See NetOrk/netork#55 and NetOrk/netork#208. VALID_VERSION_RE='^(latest|latest-dev|main-[0-9a-f]{7,40}|feature-[A-Za-z0-9._-]+|[0-9]+\.[0-9]+\.[0-9]+)$' _validate_version() { local version="$1" if [[ ! "$version" =~ $VALID_VERSION_RE ]]; then echo "Refusing to deploy version '${version}': not a tag CI publishes (expected latest," \ "latest-dev, main-, feature- (also for fix/* branches), or X.Y.Z)." >&2 return 1 fi } if [[ -n "$VERSION_OVERRIDE" ]]; then _validate_version "$VERSION_OVERRIDE" fi if [[ -z "$REGISTRY_HOST" ]]; then echo "REGISTRY_HOST is not set (deploy.env or environment): nowhere to pull images from." >&2 exit 1 fi if [[ ${#SERVERS[@]} -eq 0 ]]; then echo "No servers given: pass them as arguments or set DEPLOY_SERVERS in deploy.env." >&2 exit 1 fi # One server: under its host lock, the steps below. # # A step that fails stops the subshell (set -e) before release_host_lock runs. # That is fine: the subshell's end closes the lock session's stdin all the same. deploy_server() { local SERVER="$1" KEY="${1//./_}" VER_VAR VER_VAR="NETORK_VERSION_${KEY}" hold_host_lock "$SERVER" \ "since $(date -u +%Y-%m-%dT%H:%M:%SZ), by ${USER:-?}@$(hostname -s), deploying ${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}" deploy_steps "$SERVER" release_host_lock } deploy_steps() { local SERVER="$1" # Per-server lookup — falls back to global values local KEY="${SERVER//./_}" local USER_VAR="REGISTRY_USER_${KEY}" local PASS_VAR="REGISTRY_PASSWORD_${KEY}" local VER_VAR="NETORK_VERSION_${KEY}" local SERVER_USER="${!USER_VAR:-${REGISTRY_USER}}" local SERVER_PASS="${!PASS_VAR:-${REGISTRY_PASSWORD}}" local SERVER_VERSION="${VERSION_OVERRIDE:-${!VER_VAR:-${NETORK_VERSION}}}" _validate_version "$SERVER_VERSION" || return 1 local ENGINE_IMAGE="${REGISTRY_HOST}/netork/engine:${SERVER_VERSION}" # The password goes over ssh's stdin. Inside the ssh argument it would be # part of the remote shell's command line, readable through `ps` by every # user on the host for as long as the login runs. echo "[${SERVER}] Logging in to registry..." # shellcheck disable=SC2029 # host and user are meant to expand here printf '%s\n' "$SERVER_PASS" \ | ssh "$SERVER" "docker login '${REGISTRY_HOST}' -u '${SERVER_USER}' --password-stdin 2>&1" \ | sed "s/^/[${SERVER}] /" local REPOS="engine" if [[ " $UI_SERVER " == *" $SERVER "* ]]; then REPOS="engine ui" fi check_platform "$SERVER" "$SERVER_VERSION" "$REPOS" # The compose files are versioned with the images they start, so they are # taken from the engine image of this very tag rather than from whatever # checkout happens to run the deploy. Both are written to .new first and only # moved into place once both were read, so a failure leaves the previous pair # intact. An image built before the files moved into it has no /app/deploy/. echo "[${SERVER}] Fetching compose files from ${ENGINE_IMAGE}..." run_remote "$SERVER" "fetching compose files from ${ENGINE_IMAGE}" 'Status|Error' \ "mkdir -p ~/netork && cd ~/netork && \ docker pull '${ENGINE_IMAGE}' && \ for f in ${COMPOSE_FILES}; do \ docker run --rm --entrypoint cat '${ENGINE_IMAGE}' /app/deploy/\$f > \$f.new \ || { rm -f \$f.new; echo \"${ENGINE_IMAGE} carries no /app/deploy/\$f — the tag predates compose files in the engine image\"; exit 3; }; \ done && \ for f in ${COMPOSE_FILES}; do \ if [ -f \"\$f\" ]; then cp \"\$f\" \"\$f.prev\"; fi; mv \$f.new \$f; \ done" # Services running netOrk's own images. flower belongs here: it is the engine # image with a different command, so leaving it out let the monitoring UI run # nine days behind the workers it monitors, on an image that had since been # untagged. See NetOrk/netork#95. local SERVICES="netork-api netork-worker netork-poll-worker netork-ansible-worker netork-beat flower" if [[ " $UI_SERVER " == *" $SERVER "* ]]; then SERVICES="${SERVICES} netork-ui" fi # Containers on upstream images. Reconciled, not force-recreated (below). # postgres is deliberately absent: restarting a database on every deploy would # be worse than any compose change one could miss. redis is reconciled on its # own, before the engine (below). local INFRA_SERVICES="apt-cacher-ng signal-api" echo "[${SERVER}] Pulling images..." run_remote "$SERVER" "pulling images for ${SERVER_VERSION}" 'Pulled|Already|Error' \ "cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml pull ${SERVICES}" # The bundled database, before .env, broker and engine (remote/database.sh). A # changed image on the same volume (a minor release) is recreated; a new # volume (a major release) moves only with --infra-db, and without it the # deploy stops here, nothing changed and the old engine running. The script # goes over stdin, so the host needs nothing but bash and docker. echo "[${SERVER}] Reconciling the database (${DATABASE_MODE})..." local db_status=0 # shellcheck disable=SC2029 # mode, services and version expand here on purpose ssh "$SERVER" "REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' bash -s -- ${DATABASE_MODE} ${SERVICES}" \ < "${HERE}/remote/database.sh" 2>&1 | sed "s/^/[${SERVER}] /" || db_status=$? if [[ $db_status -eq 3 ]]; then # Refused, nothing changed: the host keeps the compose files it had, so a # hand-typed `docker compose up -d` cannot carry the database to the other # volume. .env is written only after this step. ssh -n "$SERVER" "cd ~/netork && for f in ${COMPOSE_FILES}; do if [ -f \"\$f.prev\" ]; then mv \"\$f.prev\" \"\$f\"; fi; done" || true echo "[${SERVER}] FAILED: the database step refused; the host is as it was" >&2 return 1 fi if [[ $db_status -ne 0 ]]; then echo "[${SERVER}] FAILED: the database step (its output above says what changed)" >&2 return 1 fi # docker compose substitutes REGISTRY_HOST / NETORK_VERSION from the project # .env when they are not in the environment. This script passes both inline, # but a hand-typed `docker compose up -d flower` on the host needs them there # or dies on "invalid reference format". Values are rewritten in place, so # the file does not grow a new line per deploy. # # Not earlier: this used to run before the pull, so a deploy that failed on a # tag the registry does not have still left the host recording that tag, and # the next `docker compose up` a human ran there reached for an image that # does not exist. Once the pull went through, the images are on the host. # # Not later: the services also read this file (`env_file: .env`), so it is in # every container's environment. Written after the swap, as it was for a # while, it made each container carry the previous deploy's NETORK_VERSION # (NetOrk/deploy#6). Should the swap below fail, the file names the tag whose # compose files and images are already on the host, which is what a # hand-typed `docker compose up -d` there should start. echo "[${SERVER}] Recording registry settings in ~/netork/.env..." # shellcheck disable=SC2087 # registry and version expand here; \$ escapes run remotely ssh "$SERVER" "bash -s" <> .env for kv in "REGISTRY_HOST=${REGISTRY_HOST}" "NETORK_VERSION=${SERVER_VERSION}"; do key=\${kv%%=*} if grep -q "^\${key}=" .env; then sed -i "s|^\${key}=.*|\${kv}|" .env else printf '%s\n' "\${kv}" >> .env fi done REMOTE_ENV # The broker, before the engine that cannot work without it. Plain `up -d` # recreates it only when its compose definition changed, so an unchanged # broker keeps running and keeps its queue. It used to be in no list at all, # and a changed definition -- Valkey in place of redis:7-alpine # (NetOrk/netork#1308) -- never reached a host. --wait: the engine comes up # against a healthy broker. If it does not get there, the deploy stops here, # with the old engine still running. echo "[${SERVER}] Reconciling the broker..." run_remote "$SERVER" "reconciling redis" "" \ "cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml \ up -d --no-deps --wait --wait-timeout 120 redis" # ssl_certs to netork, before the swap (NetOrk/netork#1388, #1389). netork-ui # ran nginx as root, and its entrypoint wrote the self-signed certificate into # this volume as root, the key at 0600. nginx now runs as netork, the API's # user, and cannot read that key: the UI would not start. One container of the # engine image of this version, as root with only the capability to chown, # hands the volume over. `chown -c` names only what it changed, so once a host # is handed over this does nothing and says nothing. If it fails, the deploy # stops here, with the old containers running. echo "[${SERVER}] Handing ssl_certs to netork..." run_remote "$SERVER" "handing ssl_certs to netork" "" \ "cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml \ run --rm --no-deps --user 0 --cap-add CHOWN --entrypoint chown netork-api \ -Rc netork:netork /app/ssl" # --force-recreate is deliberate: `docker compose up -d` decides what to # replace from the service definition's config hash, and the image reference # (`.../ui:latest-dev`) does not change when the tag is moved to a new image. # A deploy then leaves the old container running while reporting success — # observed on netork-ui, which kept serving a stale image after its tag had # already advanced. Recreating unconditionally costs a restart per deploy, # which a deliberate rollout wants anyway. See NetOrk/netork#90. # # Tried twice. compose recreates the services in parallel, and on 2026-10-05 it # lost a container it had just renamed ("No such container") and stopped with # every engine container Created and none running: the old API was gone, so # netOrk was down until someone ran the deploy again, which then went through # (NetOrk/netork#586). The second attempt is that rerun. Should it fail as well, # the container states go to the log and the deploy fails as before. echo "[${SERVER}] Starting containers..." local RECREATE="cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml \ up -d --no-deps --force-recreate ${SERVICES}" if ! run_remote "$SERVER" "recreating ${SERVICES}" "" "$RECREATE"; then local delay="${DEPLOY_RECREATE_RETRY_DELAY:-5}" echo "[${SERVER}] Recreating failed; trying once more in ${delay}s..." >&2 sleep "$delay" if ! run_remote "$SERVER" "recreating ${SERVICES} (second attempt)" "" "$RECREATE"; then echo "[${SERVER}] Containers after two failed attempts:" >&2 ssh -n "$SERVER" "cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -a" 2>&1 \ | sed "s/^/[${SERVER}] /" >&2 || true return 1 fi fi # Infrastructure containers, deliberately left out of the force-recreate # above. Plain `up -d`: compose compares each service definition against the # running container and recreates only what actually changed, so an edit to a # healthcheck or an option in docker-compose.yml takes effect while a # container nobody touched is left alone. signal-api is the reason this is # not --force-recreate: it holds netOrk's Signal device link and recreating # it every deploy is churn nobody asked for. See NetOrk/netork#95. # # `|| true`: infrastructure is reconciled opportunistically and its failure # is not a reason to abandon a deploy that has already succeeded — but it # prints as a failure instead of scrolling past. echo "[${SERVER}] Reconciling infrastructure containers..." run_remote "$SERVER" "reconciling ${INFRA_SERVICES}" "" \ "cd ~/netork && \ REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml \ up -d --no-deps ${INFRA_SERVICES}" || true # Services netOrk no longer ships, with the data only they used. `docker # compose up` never removes a container whose service has left the compose # file, so each would keep running on every host until someone removed it by # hand. Idempotent: a host that never had one, or was cleaned already, passes # untouched. Containers first; a volume still in use cannot be removed. # Only what exists is removed and reported: `docker rm -f` exits 0 for a # container that is not there, so without the check every deploy said it had # removed the registry, long after it was gone (NetOrk/deploy#7). # registry (netork-registry-1, netork_registry_data): the bundled registry:2 # for satellite images. Satellites pull from registry.netork.io; it held one # old image, nothing had pulled from it in 30 days, and it accepted # anonymous pushes on port 5000 (NetOrk/netork#763). local RETIRED_CONTAINERS="netork-registry-1" local RETIRED_VOLUMES="netork_registry_data" echo "[${SERVER}] Removing retired services..." run_remote "$SERVER" "removing ${RETIRED_CONTAINERS} ${RETIRED_VOLUMES}" "" \ "for c in ${RETIRED_CONTAINERS}; do \ docker inspect \$c >/dev/null 2>&1 && docker rm -f \$c >/dev/null 2>&1 \ && echo \"removed container \$c\"; done; \ for v in ${RETIRED_VOLUMES}; do \ docker volume rm \$v >/dev/null 2>&1 && echo \"removed volume \$v\"; done; true" || true # Verify the containers actually run the image we just pulled. `docker # compose up -d` reports "Running" (not "Started") when it decides nothing # changed, and `pull` prints "Pulled" even when the tag was already local — # so a deploy that had no effect at all is indistinguishable from a real one # in the output above. Both the wrong-tag case (NETORK_VERSION pointing at a # tag CI never republished) and a silently failed pull land here. # See NetOrk/netork#88. echo "[${SERVER}] Verifying containers run the pulled image..." local VERIFY_PAIRS="netork-api:engine flower:engine" if [[ " $UI_SERVER " == *" $SERVER "* ]]; then VERIFY_PAIRS="${VERIFY_PAIRS} netork-ui:ui" fi local MISMATCH MISMATCH=$(ssh -n "$SERVER" "cd ~/netork && for pair in ${VERIFY_PAIRS}; do \ svc=\${pair%%:*}; repo=\${pair##*:}; \ want=\$(docker image inspect --format '{{.Id}}' '${REGISTRY_HOST}/netork/'\${repo}':${SERVER_VERSION}' 2>/dev/null || true); \ cid=\$(REGISTRY_HOST='${REGISTRY_HOST}' NETORK_VERSION='${SERVER_VERSION}' \ docker compose -f docker-compose.yml -f docker-compose.registry.yml ps -q \${svc} 2>/dev/null || true); \ got=\$(docker inspect --format '{{.Image}}' \${cid} 2>/dev/null || true); \ if [ -z \"\${want}\" ] || [ -z \"\${got}\" ] || [ \"\${want}\" != \"\${got}\" ]; then echo \${svc}; fi; \ done" || true) if [[ -n "$MISMATCH" ]]; then echo "[${SERVER}] ERROR: still running an older image after deploy: ${MISMATCH//$'\n'/ }" >&2 echo "[${SERVER}] Wanted tag '${SERVER_VERSION}'. On main pushes CI publishes" >&2 echo "[${SERVER}] 'latest-dev' and 'main-'; ':latest' only on version tags." >&2 return 1 fi echo "[${SERVER}] Verified: containers run ${SERVER_VERSION}." # netOrk migrates *after* the swap — its API tolerates the gap. What it does # not tolerate is the migration failing unnoticed: the new code is already # serving, so a missing column is a 500 on every request that touches it # until someone looks. Loud, and non-zero. The revisions are the image's own. echo "[${SERVER}] Running migrations..." run_remote "$SERVER" "alembic upgrade head (containers are ALREADY swapped)" "" \ "cd ~/netork && docker compose exec -T netork-api alembic upgrade head" echo "[${SERVER}] Pruning unused images..." ssh -n "$SERVER" "docker image prune -af 2>&1 | tail -1" | sed "s/^/[${SERVER}] /" echo "[${SERVER}] Done." } # ── Dispatch ────────────────────────────────────────────────────────────────── echo "Deploying from ${REGISTRY_HOST} to: ${SERVERS[*]}" export -f deploy_server deploy_steps hold_host_lock release_host_lock _validate_version run_remote \ check_platform export UI_SERVER REGISTRY_HOST REGISTRY_USER REGISTRY_PASSWORD NETORK_VERSION VERSION_OVERRIDE \ VALID_VERSION_RE COMPOSE_FILES DEPLOY_LOCK_WAIT # Export all per-server credential vars so subshells can resolve them while IFS='=' read -r key _; do if [[ "$key" =~ ^(REGISTRY_(USER|PASSWORD)|NETORK_VERSION)_ ]]; then export "${key?}" fi done < <(compgen -v) PIDS=() for SERVER in "${SERVERS[@]}"; do deploy_server "$SERVER" & PIDS+=($!) done FAILED=() for i in "${!PIDS[@]}"; do if ! wait "${PIDS[$i]}"; then FAILED+=("${SERVERS[$i]}") fi done if [[ ${#FAILED[@]} -gt 0 ]]; then echo "FAILED on: ${FAILED[*]}" >&2 exit 1 fi echo "All servers deployed successfully."