diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index bb506b7c..383d0a5d 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -301,6 +301,7 @@ KEEPALIVE-CONFIG() { NODEUPDATES() { start=`date +%s` cw=40; + rm -f /tmp/.nodeupdate.* if [ "${1}" = "report" ]; then unset idsCL idsBG idsST idsCL=('') @@ -325,16 +326,33 @@ NODEUPDATES() { echo + ncount=0 for NTYPE in "${NM_NODE_TYPES[@]}"; do var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then nip='localhost ' fi - NODEUPDATE "${nip}" "${2}" "${NMCMD}" - + NODEUPDATE "${nip}" "${2}" "${NMCMD}" & + ((ncount++)) done - echo + done + + nc_count=0; completed=false + until [ "${completed}" == "true" ]; do + for nodecmd in /tmp/.nodeupdate.*.done; do + if [[ "${nodecmd}" != *"*"* ]]; then + nodecmd_output=$(cat ${nodecmd}) + [ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n" + rm -f ${nodecmd} + ((nc_count++)) + fi + done + if [ ${nc_count} -eq ${ncount} ]; then + completed=true + break + fi + sleep 2s done echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}" @@ -356,44 +374,47 @@ NODEUPDATE(){ fi NCMD="ssh -tq -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip}" NODE="${NM_HOSTNAMES[${nip}]} [${nip}]" + tmpfile=/tmp/.nodeupdate.${NM_HOSTNAMES[${nip}]// /-} + touch ${tmpfile}.running if [ ${noup} -eq 0 ]; then c=0; spca=''; spc1=`expr ${cw} - ${#NODE}`; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done - echo -en "Updating ${NODE}${spca}" + echo -en "Updating ${NODE}${spca}" >> ${tmpfile}.running fi betacheck=$(BETACHECK) if [ "${nip}" != "100.100.2.1" ] && [ "${nip}" != "10.2.1.1" ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then if [ "${betacheck}" == "false" ]; then if [ "${2}" != "sync" ]; then - [ ${noup} -eq 0 ] && echo -en " ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " + [ ${noup} -eq 0 ] && echo -en " ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " >> ${tmpfile}.running if [ "$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)" != "ok" ]; then - echo -e "${idsCL[Yellow]}New server, needs SSH credentials${idsCL[Default]}" + echo -e "${idsCL[Yellow]}[${nip}] New server, needs SSH credentials${idsCL[Default]}" ssh-copy-id root@${nip} fi if ! ${NCMD} command -v nodemgmt > /dev/null; then ${NCMD} "bash <(curl -sL https://go.scity.us/install-nodemgmt)" >/dev/null 2>&1 ${NMCMD} rsync -az ${NM_FOLDER}/conf/*.inc ${NM_FOLDER}/conf/*.conf root@${nip}:${NM_FOLDER}/conf/ & >/dev/null 2>&1 [ "${NM_GITPASS}" != "" ] && [ "${NM_GITUSER}" == "voltron" ] && ${NMCMD} "bash <(curl -sL https://go.scity.us/add-linux-pass)" >/dev/null 2>&1 - echo -e "${idsCL[Green]}Installed & Synced${idsCL[Default]}" + echo -e "${idsCL[Green]}Installed & Synced${idsCL[Default]}" >> ${tmpfile}.running elif [ ${noup} -eq 1 ]; then - echo -e "${idsCL[Green]}Already Installed${idsCL[Default]}" + echo -e "${idsCL[Green]}Already Installed${idsCL[Default]}" >> ${tmpfile}.running else - ${NCMD} "nodemgmt update q" + ${NCMD} "nodemgmt update q" >> ${tmpfile}.running fi else - echo -en " ~ ${idsCL[LightCyan]}Sync Shared Nodemgmt Defaults${idsCL[Default]} : " + echo -en " ~ ${idsCL[LightCyan]}Sync Shared Nodemgmt Defaults${idsCL[Default]} : " >> ${tmpfile}.running ${NMCMD} rsync -az ${NM_FOLDER}/conf/*.inc ${NM_FOLDER}/conf/*.conf root@${nip}:${NM_FOLDER}/conf/ & >/dev/null 2>&1 - echo -e "${idsCL[Green]}Files Synced${idsCL[Default]}" + echo -e "${idsCL[Green]}Files Synced${idsCL[Default]}" >> ${tmpfile}.running fi else - echo -e " ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : Skipping BETA Update" + echo -e " ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : Skipping BETA Update" >> ${tmpfile}.running fi elif [ -f ${NM_TMPFOLDER}/${nip}.down ]; then - echo -e " ~ ${idsCL[LightCyan]}Skipping, host is offline${idsCL[Default]}" + echo -e " ~ ${idsCL[LightCyan]}Skipping, host is offline${idsCL[Default]}" >> ${tmpfile}.running else - echo -e " ~ ${idsCL[LightCyan]}Skipping${idsCL[Default]}" + echo -e " ~ ${idsCL[LightCyan]}Skipping${idsCL[Default]}" >> ${tmpfile}.running fi + mv ${tmpfile}.running ${tmpfile}.done } ADD_LOGROTATE_CRONTAB(){