diff --git a/docs/deployment.md b/docs/deployment.md index 83ba85a..e1e06f4 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -116,6 +116,17 @@ curl --fail "http://${TAPM_LISTEN_ADDR}/health/ready" There is no local application state and no session affinity requirement. +For later updates, run the repository update helper on one webserver at a time: + +```sh +/opt/idssys/ta-deployment-broker/update.sh +``` + +The helper fast-forwards the current tracked branch, rebuilds the image, applies +any pending migrations, recreates the broker container, and waits up to 150 +seconds for its Docker health check. It refuses to pull over tracked local +changes or from a detached/untracked branch. + ## 7. HAProxy health check The load balancer should consider an origin healthy only when diff --git a/update.sh b/update.sh new file mode 100755 index 0000000..58dd3bb --- /dev/null +++ b/update.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +SCRIPT_PATH="${SCRIPT_DIR}/$(basename -- "${BASH_SOURCE[0]}")" + +fail() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +trap 'printf "ERROR: broker update failed at line %d.\n" "$LINENO" >&2' ERR + +cd "$SCRIPT_DIR" + +command -v git >/dev/null 2>&1 || fail "git is required" +command -v docker >/dev/null 2>&1 || fail "docker is required" +docker compose version >/dev/null 2>&1 || fail "Docker Compose v2 is required" +[[ -f compose.yaml ]] || fail "compose.yaml was not found in ${SCRIPT_DIR}" +[[ -f .env ]] || fail ".env was not found in ${SCRIPT_DIR}" + +if [[ "${1:-}" != "--post-pull" ]]; then + branch="$(git symbolic-ref --quiet --short HEAD)" || + fail "the repository is in detached HEAD state" + git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' >/dev/null 2>&1 || + fail "branch ${branch} does not have an upstream branch" + git diff --quiet && git diff --cached --quiet || + fail "tracked local changes must be committed or stashed before updating" + + printf 'Updating TAPM Deployment Broker from branch %s...\n' "$branch" + git pull --ff-only + + # Restart from the newly pulled script so updates to this file take effect. + exec "$SCRIPT_PATH" --post-pull +fi + +printf 'Building broker image...\n' +docker compose build broker + +printf 'Applying pending database migrations...\n' +docker compose run --rm migrate + +printf 'Starting broker...\n' +docker compose up -d broker + +container_id="$(docker compose ps -q broker)" +[[ -n "$container_id" ]] || fail "Docker Compose did not return a broker container" + +printf 'Waiting for broker health check' +for ((attempt = 1; attempt <= 75; attempt++)); do + status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")" + case "$status" in + healthy) + printf ' healthy.\n' + docker compose ps broker + printf 'TAPM Deployment Broker update completed successfully.\n' + exit 0 + ;; + exited | dead | unhealthy) + printf ' %s.\n' "$status" + docker compose logs --tail=100 broker + fail "broker failed its health check" + ;; + esac + printf '.' + sleep 2 +done + +printf ' timed out.\n' +docker compose logs --tail=100 broker +fail "broker did not become healthy within 150 seconds"