update

docs/deployment.md

@@ -116,18 +116,24 @@ curl --fail "http://${TAPM_LISTEN_ADDR}/health/ready"
 
 There is no local application state and no session affinity requirement.
 
-For later updates, run the repository update helper on one webserver at a time:
+For later updates, run the repository update helper from either webserver:
 
 ```sh
 /opt/idssys/ta-deployment-broker/update.sh
 ```
 
-The helper fetches and compares the current tracked branch with its upstream. If
+The helper updates and health-checks the local broker first, then connects as
-both commits match, it exits without touching Docker. When an update exists, it
+`root` to the other broker (`10.10.1.121` or `10.10.1.122`) and performs the
-fast-forwards the branch, rebuilds the image, applies any pending migrations,
+same local-only check there. This provides a sequential rolling update from
-recreates the broker container, and waits up to 150 seconds for its Docker
+either node. An unreachable peer is skipped with a warning; a reachable peer
-health check. It refuses to pull over tracked local changes or from a detached,
+whose update fails causes the command to fail.
-untracked, locally-ahead, or diverged branch.
+
+Each node fetches and compares its current tracked branch with its upstream. If
+both commits match, that node's Docker services are untouched. When an update
+exists, the helper fast-forwards the branch, rebuilds the image, applies pending
+migrations, recreates the broker container, and waits up to 150 seconds for its
+Docker health check. It refuses tracked local changes and detached, untracked,
+locally-ahead, or diverged branches.
 
 ## 7. HAProxy health check
 

update.sh

@@ -16,38 +16,17 @@ cd "$SCRIPT_DIR"
 
 command -v git >/dev/null 2>&1 || fail "git is required"
 
-if [[ "${1:-}" != "--post-pull" ]]; then
+POST_PULL=0
-	branch="$(git symbolic-ref --quiet --short HEAD)" ||
+LOCAL_ONLY="${TAPM_UPDATE_LOCAL_ONLY:-0}"
-		fail "the repository is in detached HEAD state"
+for argument in "$@"; do
-	upstream="$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' 2>/dev/null)" ||
+	case "$argument" in
-		fail "branch ${branch} does not have an upstream branch"
+		--post-pull) POST_PULL=1 ;;
-	remote="$(git config --get "branch.${branch}.remote")" ||
+		--local-only) LOCAL_ONLY=1 ;;
-		fail "branch ${branch} does not have a configured remote"
+		*) fail "unknown argument: ${argument}" ;;
-	git diff --quiet && git diff --cached --quiet ||
+	esac
-		fail "tracked local changes must be committed or stashed before updating"
+done
-
-	printf 'Checking %s against %s...\n' "$branch" "$upstream"
-	git fetch --prune "$remote"
-
-	local_commit="$(git rev-parse HEAD)"
-	upstream_commit="$(git rev-parse '@{upstream}')"
-	if [[ "$local_commit" == "$upstream_commit" ]]; then
-		printf 'TAPM Deployment Broker is already up to date; no changes were made.\n'
-		exit 0
-	fi
-	if git merge-base --is-ancestor "$local_commit" "$upstream_commit"; then
-		printf 'Updating TAPM Deployment Broker from %s...\n' "$upstream"
-		git merge --ff-only "$upstream_commit"
-	elif git merge-base --is-ancestor "$upstream_commit" "$local_commit"; then
-		fail "local branch ${branch} is ahead of ${upstream}; refusing to rebuild"
-	else
-		fail "local branch ${branch} has diverged from ${upstream}"
-	fi
-
-	# Restart from the newly pulled script so updates to this file take effect.
-	exec "$SCRIPT_PATH" --post-pull
-fi
 
+deploy_local_broker() {
 	command -v docker >/dev/null 2>&1 || fail "docker is required"
 	docker compose version >/dev/null 2>&1 || fail "Docker Compose v2 is required"
 	[[ -f compose.yaml ]] || fail "compose.yaml was not found in ${SCRIPT_DIR}"
@@ -72,8 +51,8 @@ for ((attempt = 1; attempt <= 75; attempt++)); do
 			healthy)
 				printf ' healthy.\n'
 				docker compose ps broker
-			printf 'TAPM Deployment Broker update completed successfully.\n'
+				printf 'Local TAPM Deployment Broker update completed successfully.\n'
-			exit 0
+				return 0
 				;;
 			exited | dead | unhealthy)
 				printf ' %s.\n' "$status"
@@ -88,3 +67,88 @@ done
 	printf ' timed out.\n'
 	docker compose logs --tail=100 broker
 	fail "broker did not become healthy within 150 seconds"
+}
+
+update_peer_broker() {
+	local listen_addr
+	local local_address
+	local peer_host="${TAPM_UPDATE_PEER_HOST:-}"
+	local ssh_user="${TAPM_UPDATE_SSH_USER:-root}"
+	local remote_update_path='/opt/idssys/ta-deployment-broker/update.sh'
+	local -a ssh_options=(
+		-o BatchMode=yes
+		-o ConnectTimeout=5
+		-o ConnectionAttempts=1
+		-o StrictHostKeyChecking=yes
+	)
+
+	[[ -f .env ]] || fail ".env was not found in ${SCRIPT_DIR}"
+	if [[ -z "$peer_host" ]]; then
+		listen_addr="$(
+			sed -n 's/^[[:space:]]*TAPM_LISTEN_ADDR[[:space:]]*=[[:space:]]*//p' .env |
+				tail -n 1
+		)"
+		listen_addr="${listen_addr#\"}"
+		listen_addr="${listen_addr%\"}"
+		listen_addr="${listen_addr#\'}"
+		listen_addr="${listen_addr%\'}"
+		local_address="${listen_addr%:*}"
+		case "$local_address" in
+			10.10.1.121) peer_host='10.10.1.122' ;;
+			10.10.1.122) peer_host='10.10.1.121' ;;
+			*) fail "unable to determine peer from TAPM_LISTEN_ADDR=${listen_addr}" ;;
+		esac
+	fi
+
+	printf 'Checking peer broker at %s...\n' "$peer_host"
+	if ! ssh "${ssh_options[@]}" "${ssh_user}@${peer_host}" true; then
+		printf 'WARNING: peer %s is unavailable over SSH; its update was skipped.\n' "$peer_host" >&2
+		return 0
+	fi
+	if ! ssh "${ssh_options[@]}" "${ssh_user}@${peer_host}" \
+		"TAPM_UPDATE_LOCAL_ONLY=1 ${remote_update_path} --local-only"; then
+		fail "peer ${peer_host} is reachable, but its broker update failed"
+	fi
+	printf 'Peer broker at %s completed successfully.\n' "$peer_host"
+}
+
+if ((POST_PULL == 0)); then
+	branch="$(git symbolic-ref --quiet --short HEAD)" ||
+		fail "the repository is in detached HEAD state"
+	upstream="$(git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' 2>/dev/null)" ||
+		fail "branch ${branch} does not have an upstream branch"
+	remote="$(git config --get "branch.${branch}.remote")" ||
+		fail "branch ${branch} does not have a configured remote"
+	git diff --quiet && git diff --cached --quiet ||
+		fail "tracked local changes must be committed or stashed before updating"
+
+	printf 'Checking %s against %s...\n' "$branch" "$upstream"
+	git fetch --prune "$remote"
+
+	local_commit="$(git rev-parse HEAD)"
+	upstream_commit="$(git rev-parse '@{upstream}')"
+	if [[ "$local_commit" == "$upstream_commit" ]]; then
+		printf 'Local TAPM Deployment Broker is already up to date.\n'
+	elif git merge-base --is-ancestor "$local_commit" "$upstream_commit"; then
+		printf 'Updating TAPM Deployment Broker from %s...\n' "$upstream"
+		git merge --ff-only "$upstream_commit"
+
+		# Restart from the newly pulled script so updates to this file take effect.
+		if ((LOCAL_ONLY == 1)); then
+			exec "$SCRIPT_PATH" --post-pull --local-only
+		fi
+		exec "$SCRIPT_PATH" --post-pull
+	elif git merge-base --is-ancestor "$upstream_commit" "$local_commit"; then
+		fail "local branch ${branch} is ahead of ${upstream}; refusing to rebuild"
+	else
+		fail "local branch ${branch} has diverged from ${upstream}"
+	fi
+else
+	deploy_local_broker
+fi
+
+if ((LOCAL_ONLY == 0)); then
+	update_peer_broker
+fi
+
+printf 'TAPM Deployment Broker update check completed.\n'