diff --git a/defaults.inc b/defaults.inc index 1ae60440..16607705 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='4.6.12-04302023' +VERS='4.6.23-04302023' #NODETYPES=WEB,LB NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW @@ -32,10 +32,14 @@ WEBDOCKER=phpmyadmin,phppgadmin noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck ' CERT_DAEMON='/snap/bin/certbot' FOLDER=/opt/idssys/nodemgmt +TMPFOLDER=${FOLDER}/.tmp SCRIPT=${FOLDER}/nodemgmt-scripts.sh LOGFILE=${FOLDER}/logfile RENOTIFY=900 +[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER} + + NM_SRVCOPT=(start stop restart reload enable disable) #MYSQL_REPL_CHECK=( ) diff --git a/inc/status.inc b/inc/status.inc index 29a53298..4e7805e7 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -412,71 +412,70 @@ STATUS(){ echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" DIVIDER; echo -e "${idsST[Reset]}" + echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : " fi - # replcheckstart=`date +%s` - # echo "$(date) - STARTING REPL CHECK - $(SHOW_TIME $(expr `date +%s` - $replstart) s)" >> ${FOLDER}/log.repl for NTYPE in "${NODE_TYPES[@]}"; do PH=${NTYPE}_HOSTS[0] PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then - nid=1 - var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do - - # echo "CHECK: ${NTYPE} - ${nip}" - + if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then + if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then + var2=${NTYPE}_REPL_CHECK[@] + for rcheck in "${!var2}"; do + REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 + done + fi + fi + done + fi + done + [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + + for NTYPE in "${NODE_TYPES[@]}"; do + PH=${NTYPE}_HOSTS[0] + PH_CMD="ssh root@${!PH}" + var=${NTYPE}_REPL_CHECK[@] + if [ ! -z ${!var+x} ]; then + var=${NTYPE}_HOSTS[@] + for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi - - # checkhost=$(CHECK_HOST ${nip}) - #if [ "${checkhost}" != "false" ]; then if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] - # timeout=`date --date='20 seconds' +%s` for rcheck in "${!var2}"; do if [ "${ST_ACTION}" != "check" ]; then - [ "${ST_ACTION}" != "report" ] && c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done || spc=' ' + if [ "${ST_ACTION}" != "report" ]; then + c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + else + spc=' ' + fi echo -en " ${REPL_DESC[${rcheck}]}${spc}: " - timeout=`date --date='2 minutes' +%s` - warn_timeout=`date --date='30 secs' +%s` - else - timeout=`date --date='2 minutes' +%s` + fi - checked=false - # replnodestart=`date +%s` - until [ "${checked}" = "" ]; do - if [ "${PH_CMD}" = "" ]; then - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` - else - ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" - fi - if [ "`date +%s`" -gt "${timeout}" ]; then - timeout=true - break - elif [ "`date +%s`" -gt "${warn_timeout}" ] && [ "${ST_ACTION}" != "report" ]; then - echo -en " waiting for sync... " - warn_timeout=`date --date='100 minutes' +%s` + checked="" + until [ "${checked}" != "" ]; do + if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then + checked=good + elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then + checked=timeout fi + rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.* done - # replend=`date +%s` - # replruntime=$((replend-replstart)) - # replnoderuntime=$((replend-replnodestart)) - # replcheckruntime=$((replend-replcheckstart)) - # rrmsg="$(SHOW_TIME ${replruntime} s) (copy) $(SHOW_TIME ${replcheckruntime} s) (start) $(SHOW_TIME ${replnoderuntime} s) (node)" - if [ "${timeout}" = "true" ]; then + + if [ "${checked}" == "timeout" ]; then [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then touch ${FOLDER}/${nip}-${rcheck}.down SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} fi - # echo "$(date) - TIMEOUT - ${rcheck} - ${rrmsg}" >> ${FOLDER}/log.repl else [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then @@ -484,9 +483,7 @@ STATUS(){ SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} fi - # echo "$(date) - SYNCED - ${rcheck} - ${rrmsg}" >> ${FOLDER}/log.repl - fi - + fi done else if [ "${ST_ACTION}" != "check" ]; then @@ -496,7 +493,6 @@ STATUS(){ fi if [ "${ST_ACTION}" != "check" ]; then echo; fi fi - nid=`expr $nid + 1` done fi done @@ -664,4 +660,26 @@ STATUS(){ end=`date +%s` runtime=$((end-start)) echo "runtime: ${runtime}" +} + + +REPLCHECK(){ + rcheck=${1} + nip=${2} + PH_CMD=${3} + if [ "${4}" != "check" ]; then + timeout=`date --date='1 minutes' +%s` + else + timeout=`date --date='2 minutes' +%s` + fi + checked=false + until [ "${checked}" == "" ]; do + if [ "${PH_CMD}" == "" ]; then + ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` + else + ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" + fi + [ "`date +%s`" -gt "${timeout}" ] && timeout=true && break + done + [ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good } \ No newline at end of file diff --git a/run.sh b/run.sh index 788161b8..542ba85e 100755 --- a/run.sh +++ b/run.sh @@ -36,13 +36,16 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then git submodule update --remote >/dev/null 2>&1 fi cd ${FOLDER} - if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then + # if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then + if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then if [ "${2}" != "q" ]; then echo -en "\e[1A"; echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}" fi - git fetch origin master >/dev/null 2>&1 - git reset --hard origin/master >/dev/null 2>&1 + # git fetch origin master >/dev/null 2>&1 + # git reset --hard origin/master >/dev/null 2>&1 + git fetch origin dev >/dev/null 2>&1 + git reset --hard origin/dev >/dev/null 2>&1 git reflog expire --expire=now --all >/dev/null 2>&1 git repack -ad >/dev/null 2>&1 git prune >/dev/null 2>&1