voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
David Schroeder
2023-05-01 22:00:10 -05:00
parent 20a96bc828 40b5ce7baf
commit 9b3c80e5da
3 changed files with 70 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
defaults.inc
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
VERS='4.6.12-04302023' VERS='4.6.23-04302023'
#NODETYPES=WEB,LB #NODETYPES=WEB,LB
NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW NODETYPES=MYSQL,LB,WEB,NC,WPD,LPD,MM,PW
@@ -32,10 +32,14 @@ WEBDOCKER=phpmyadmin,phppgadmin
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck ' noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck '
CERT_DAEMON='/snap/bin/certbot' CERT_DAEMON='/snap/bin/certbot'
FOLDER=/opt/idssys/nodemgmt FOLDER=/opt/idssys/nodemgmt
TMPFOLDER=${FOLDER}/.tmp
SCRIPT=${FOLDER}/nodemgmt-scripts.sh SCRIPT=${FOLDER}/nodemgmt-scripts.sh
LOGFILE=${FOLDER}/logfile LOGFILE=${FOLDER}/logfile
RENOTIFY=900 RENOTIFY=900
[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
NM_SRVCOPT=(start stop restart reload enable disable) NM_SRVCOPT=(start stop restart reload enable disable)
#MYSQL_REPL_CHECK=( ) #MYSQL_REPL_CHECK=( )

96
inc/status.inc
View File

@@ -412,71 +412,70 @@ STATUS(){
echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}" DIVIDER; echo -e "${idsST[Reset]}"
echo -en " ${idsCL[LightCyan]}Starting processes to collect/monitor replication status : "
fi fi
# replcheckstart=`date +%s`
# echo "$(date) - STARTING REPL CHECK - $(SHOW_TIME $(expr `date +%s` - $replstart) s)" >> ${FOLDER}/log.repl
for NTYPE in "${NODE_TYPES[@]}"; do for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0] PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}" PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@] var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then if [ ! -z ${!var+x} ]; then
nid=1
var=${NTYPE}_HOSTS[@] var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
done
fi
fi
done
fi
done
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
# echo "CHECK: ${NTYPE} - ${nip}" for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green DIVIDER false green
fi fi
# checkhost=$(CHECK_HOST ${nip})
#if [ "${checkhost}" != "false" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${FOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@] var2=${NTYPE}_REPL_CHECK[@]
# timeout=`date --date='20 seconds' +%s`
for rcheck in "${!var2}"; do for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
[ "${ST_ACTION}" != "report" ] && c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done || spc=' ' if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: " echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
timeout=`date --date='2 minutes' +%s`
warn_timeout=`date --date='30 secs' +%s`
else
timeout=`date --date='2 minutes' +%s`
fi fi
checked=false checked=""
# replnodestart=`date +%s` until [ "${checked}" != "" ]; do
until [ "${checked}" = "" ]; do if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
if [ "${PH_CMD}" = "" ]; then checked=good
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
else checked=timeout
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "${timeout}" ]; then
timeout=true
break
elif [ "`date +%s`" -gt "${warn_timeout}" ] && [ "${ST_ACTION}" != "report" ]; then
echo -en " waiting for sync... "
warn_timeout=`date --date='100 minutes' +%s`
fi fi
rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
done done
# replend=`date +%s`
# replruntime=$((replend-replstart)) if [ "${checked}" == "timeout" ]; then
# replnoderuntime=$((replend-replnodestart))
# replcheckruntime=$((replend-replcheckstart))
# rrmsg="$(SHOW_TIME ${replruntime} s) (copy) $(SHOW_TIME ${replcheckruntime} s) (start) $(SHOW_TIME ${replnoderuntime} s) (node)"
if [ "${timeout}" = "true" ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down touch ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi fi
# echo "$(date) - TIMEOUT - ${rcheck} - ${rrmsg}" >> ${FOLDER}/log.repl
else else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
@@ -484,9 +483,7 @@ STATUS(){
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi fi
# echo "$(date) - SYNCED - ${rcheck} - ${rrmsg}" >> ${FOLDER}/log.repl
fi fi
done done
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
@@ -496,7 +493,6 @@ STATUS(){
fi fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi fi
nid=`expr $nid + 1`
done done
fi fi
done done
@@ -665,3 +661,25 @@ STATUS(){
runtime=$((end-start)) runtime=$((end-start))
echo "runtime: ${runtime}" echo "runtime: ${runtime}"
} }
REPLCHECK(){
rcheck=${1}
nip=${2}
PH_CMD=${3}
if [ "${4}" != "check" ]; then
timeout=`date --date='1 minutes' +%s`
else
timeout=`date --date='2 minutes' +%s`
fi
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done
[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
}

9
run.sh
View File

@@ -36,13 +36,16 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
git submodule update --remote >/dev/null 2>&1 git submodule update --remote >/dev/null 2>&1
fi fi
cd ${FOLDER} cd ${FOLDER}
if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then # if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then
if [ "${2}" != "q" ]; then if [ "${2}" != "q" ]; then
echo -en "\e[1A"; echo -en "\e[1A";
echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}" echo -e "\e[0K\r ${idsCL[LightCyan]}Installing updates...${idsCL[Default]}"
fi fi
git fetch origin master >/dev/null 2>&1 # git fetch origin master >/dev/null 2>&1
git reset --hard origin/master >/dev/null 2>&1 # git reset --hard origin/master >/dev/null 2>&1
git fetch origin dev >/dev/null 2>&1
git reset --hard origin/dev >/dev/null 2>&1
git reflog expire --expire=now --all >/dev/null 2>&1 git reflog expire --expire=now --all >/dev/null 2>&1
git repack -ad >/dev/null 2>&1 git repack -ad >/dev/null 2>&1
git prune >/dev/null 2>&1 git prune >/dev/null 2>&1