From e05c11473fb04dc28778a48835782ab4b37879ce Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Tue, 12 Dec 2023 09:07:27 -0600 Subject: [PATCH] update --- defaults.inc | 2 +- inc/certs.inc | 16 ++--- inc/services.inc | 87 ------------------------- inc/status.inc | 154 ++++++++++++++++++++++---------------------- nodemgmt-scripts.sh | 39 +++++------ 5 files changed, 106 insertions(+), 192 deletions(-) diff --git a/defaults.inc b/defaults.inc index 56b4b257..67f510cd 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ #!/usr/bin/env bash -VERS='5.1.12-12122023' +VERS='5.1.13-12122023' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck ' diff --git a/inc/certs.inc b/inc/certs.inc index 8635ed57..3f2cf3f6 100755 --- a/inc/certs.inc +++ b/inc/certs.inc @@ -86,7 +86,7 @@ NEWCERT(){ fi echo - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -145,7 +145,7 @@ CERTRENEW(){ SERVICE_MGMT nginx reload >> ${NM_LOGFOLDER}/cert-renewal.lastrun echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}" echo - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -205,7 +205,7 @@ LISTCERTS(){ rm -f /tmp/ssllist # #for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done > /tmp/ssllist # fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true fi echo -e "\033[K${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" @@ -441,7 +441,7 @@ LISTCERTS(){ echo - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -468,7 +468,7 @@ LISTCERTS_NPM(){ rm -f /tmp/ssllist for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done > /tmp/ssllist fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true fi echo @@ -516,7 +516,7 @@ LISTCERTS_NPM(){ DIVIDER false darkGray 120 done echo - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -539,7 +539,7 @@ COPYCERTS_NPM(){ done fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true fi @@ -552,7 +552,7 @@ COPYCERTS_NPM(){ fi done - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then echo DIVIDER true ENTER2CONTINUE diff --git a/inc/services.inc b/inc/services.inc index 98936bc4..b74f216d 100755 --- a/inc/services.inc +++ b/inc/services.inc @@ -445,93 +445,6 @@ SERVICE_MGMT(){ fi } -CHECK_DOCKER_SERVICES(){ - - for NTYPE in "${NM_NODE_TYPES[@]}"; do - dockers=${NTYPE}_DOCKERS_CHECK[@] - hosts=${NTYPE}_HOSTS[@] - - if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then - if [ "${NMG_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - - for nip in "${!hosts}"; do - if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - fi - if [ "${checkhost}" != "false" ]; then - if [ "${NMG_ACTION}" != "check" ]; then - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" - uptime=`${NCMD} uptime -p` - echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - DIVIDER false green - fi - if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then - if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - else - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - fi - rm -f ${NM_TMPFOLDER}/${nip}.down - rm -f ${NM_TMPFOLDER}/${nip}.errtime - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - - for docker in "${!dockers}"; do - if [ "${NMG_ACTION}" != "check" ]; then - c=0; cw=30; spc='' - spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: " - fi - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then - ${NCMD} docker rm ${docker} - fi - ${NCMD} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${NMG_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" - fi - else - if [ "${NMG_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - fi - fi - else - if [ "${NMG_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - fi - done - else - if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - touch ${NM_TMPFOLDER}/${nip}.down - if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - touch ${NM_TMPFOLDER}/${nip}.errtime - fi - else - errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) - fi - if [ "${NMG_ACTION}" != "check" ]; then - toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" - fi - fi - if [ "${NMG_ACTION}" != "check" ]; then echo; fi - done - fi - done - -} - diff --git a/inc/status.inc b/inc/status.inc index d6e796aa..80a35863 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -4,16 +4,16 @@ STATUS(){ log_start=$(date "+%Y-%m-%d %H:%M:%S") # if [ "${1}" == "report" ] || [ "${1}" == "" ]; then - # NMG_ACTION=${1} + # STATUS_ACTION=${1} # ntypesel=(${NM_NODE_TYPES[@]}) - # # if [ "${NM_NODETYPES[${NMG_ACTION^^}]}" != "" ]; then - # # ntss=${NMG_ACTION^^} + # # if [ "${NM_NODETYPES[${STATUS_ACTION^^}]}" != "" ]; then + # # ntss=${STATUS_ACTION^^} # # ntypesel=(${ntss}) - # # NMG_ACTION=${2} + # # STATUS_ACTION=${2} # # else # # ntss="" # # ntypesel=(${NM_NODE_TYPES[@]}) - # # [ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1} + # # [ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1} # # fi # else # while [ $# -gt 0 ]; do @@ -38,11 +38,11 @@ STATUS(){ if [ "${NM_NODETYPES[${1^^}]}" != "" ]; then ntss=${1^^} ntypesel=(${ntss}) - NMG_ACTION=${2} + STATUS_ACTION=${2} else ntss="" ntypesel=(${NM_NODE_TYPES[@]}) - [ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1} + [ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1} fi PRI_CW=40 @@ -55,15 +55,15 @@ STATUS(){ ############################ ([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge 300 ]) && REPLRUN=1 || REPLRUN=0 - # if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then - if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then - if [ "${NMG_ACTION}" != "check" ]; then + # if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then + if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}" fi for NTYPE in "${ntypesel[@]}"; do REPLSTART ${NTYPE} & done - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n" echo -en "${idsCL[LightCyan]}Starting processes to collect/monitor replication check : " fi @@ -78,38 +78,38 @@ STATUS(){ if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do - REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${NMG_ACTION}" & >/dev/null 2>&1 + REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${STATUS_ACTION}" & >/dev/null 2>&1 done fi fi done fi done - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n" fi - if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "services" ] || [ "${NMG_ACTION}" == "dockers" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "all" ]; then + if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "services" ] || [ "${STATUS_ACTION}" == "dockers" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "all" ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : " + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : " for NTYPE in "${ntypesel[@]}"; do STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running 2>/dev/null & done - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n" ######################## ## LOCALHOST CHECK ######################## - if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${NMG_ACTION}" != "dockers" ] && [ "${NMG_ACTION}" != "services" ] && [ "${ntss}" == "" ]; then + if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${STATUS_ACTION}" != "dockers" ] && [ "${STATUS_ACTION}" != "services" ] && [ "${ntss}" == "" ]; then cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then lip=${RUN_NODE_IP} echo -e "\n$(DIVIDER)\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}" echo -e "$(DIVIDER)" echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} ${idsST[Reset]}${idsCL[LightCyan]}[${lip}]" - if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then + if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then uptime=`uptime -p` echo -e "${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}" else @@ -118,15 +118,15 @@ STATUS(){ DIVIDER . green fi - # if [ "${NMG_ACTION}" != "check" ]; then + # if [ "${STATUS_ACTION}" != "check" ]; then # lip=${RUN_NODE_IP} # uptime=`uptime -p` # echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" # DIVIDER false green # fi for srvc in "${LOCAL_SERVICES[@]}"; do - if [ "${NMG_ACTION}" != "check" ]; then - if [ "${NMG_ACTION}" != "report" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "report" ]; then c=0; cw=${PRI_CW}; spc='' spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -138,7 +138,7 @@ STATUS(){ if [ "$(systemctl is-active ${srvc})" != "active" ]; then if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} @@ -147,12 +147,12 @@ STATUS(){ if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)) - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down fi fi else - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}" if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)) @@ -204,9 +204,9 @@ STATUS(){ ## REPLICATION CHECK ######################## - # if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then - if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then - if [ "${NMG_ACTION}" != "check" ]; then + # if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then + if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" DIVIDER; echo -en "${idsST[Reset]}" @@ -220,15 +220,15 @@ STATUS(){ var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do - if [ "${NMG_ACTION}" != "check" ]; then - if [ "${NMG_ACTION}" != "report" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "report" ]; then c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done else spc=' ' @@ -247,7 +247,7 @@ STATUS(){ done if [ "${checked}" == "timeout" ]; then - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down else @@ -262,7 +262,7 @@ It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDE fi elif [ "${checked}" == "good" ]; then - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}" if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) @@ -276,12 +276,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ done else - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then [ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}" [ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}" fi fi - [ "${NMG_ACTION}" != "check" ] && echo + [ "${STATUS_ACTION}" != "check" ] && echo fi @@ -291,7 +291,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ ########################## # REMOVE REPL CHECK FILES ########################## - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}" for NTYPE in "${ntypesel[@]}"; do PH=${NTYPE}_HOSTS[0] if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then @@ -312,16 +312,16 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ done rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl touch ${NM_TMPFOLDER}/.replcheck - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n" fi ######################## ## FREE SPACE CHECK ######################## - #if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "freespace" ]; then - if [ "${NMG_ACTION}" == "freespace" ]; then + #if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "freespace" ]; then + if [ "${STATUS_ACTION}" == "freespace" ]; then for NTYPE in "${ntypesel[@]}"; do - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}" DIVIDER; echo -e "${idsST[Reset]}" @@ -331,12 +331,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH='' fi - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}" DIVIDER false green fi - [ "${NMG_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" declare -A partitions DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') DRIVEINFO=(${DRIVEINFO}) @@ -352,7 +352,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ pname=${pname#*vg-} if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then freespace=${DRIVEINFO[$((${ii}+2))]//M/} - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then c=0; cw=20; spc='' spc1=${cw}-${#pname} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -376,7 +376,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ fs_status='' fs_status_color='Green' fi - [ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}" fi done @@ -384,7 +384,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ # if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then # pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1 # pname=${pname#*vg-} - # if [ "${NMG_ACTION}" != "check" ]; then + # if [ "${STATUS_ACTION}" != "check" ]; then # c=0; cw=20; spc='' # spc1=${cw}-${#pname} # until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -409,7 +409,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ # fs_status_color='Green' # # fi - # if [ "${NMG_ACTION}" != "check" ]; then + # if [ "${STATUS_ACTION}" != "check" ]; then # if [[ ${freespace} -gt 1000 ]]; then # fsgb=$(bc <<< "scale=2; ${freespace}/1000") # fsdsp="${fsgb} GB" @@ -427,9 +427,9 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ done fi ######################## - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo - if [ -z ${ACTION} ] || [ "${ACTION}" == "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" == "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -437,7 +437,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ end=`date +%s` runtime=$((end-start)) echo -e "Runtime: ${runtime}\n" - [ "${NMG_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes + [ "${STATUS_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes } STATUS_NODE(){ @@ -447,7 +447,7 @@ STATUS_NODE(){ NTHOSTS=${NODETYPE}_HOSTS[@] - if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${NMG_ACTION}" == "services" ]); then + if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then GOFORCHECK=false; else GOFORCHECK=true; @@ -455,9 +455,9 @@ STATUS_NODE(){ if [ ${GOFORCHECK} = true ]; then - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER - if [ "${NMG_ACTION}" == "dockers" ]; then + if [ "${STATUS_ACTION}" == "dockers" ]; then echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" else echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" @@ -465,9 +465,9 @@ STATUS_NODE(){ DIVIDER . lightYellow; echo -en "${idsST[Reset]}" fi for nip in "${!NTHOSTS}"; do - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}" - [ "${NMG_ACTION}" != "report" ] && echo -en " - Verifying Host..." + [ "${STATUS_ACTION}" != "report" ] && echo -en " - Verifying Host..." fi if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then @@ -486,8 +486,8 @@ STATUS_NODE(){ [ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage="" - if [ "${NMG_ACTION}" != "check" ]; then - if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then if [ "${NODETYPE}" == "OFW" ]; then uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1) if [ ${#uptime} -ge 6 ]; then @@ -501,7 +501,7 @@ STATUS_NODE(){ else uptime=$(${NCMD} uptime -p) fi - [ "${NMG_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}" + [ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}" else echo -e "${idsCL[Default]}" fi @@ -520,7 +520,7 @@ STATUS_NODE(){ ######################## ## SERVICES CHECK ######################## - if [ "${NMG_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then + if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}" DIVIDER . green 52 @@ -529,7 +529,7 @@ STATUS_NODE(){ [ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`) sr=0 for srvc in "${!srvcs}"; do - if [ "${NMG_ACTION}" != "report" ]; then + if [ "${STATUS_ACTION}" != "report" ]; then c=0; cw=${PRI_CW}; spc='' spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -551,7 +551,7 @@ STATUS_NODE(){ [ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false if [ ${SSCHK} != false ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " if [ "${NODETYPE}" != "OFW" ]; then srvctest=${srvctst[$sr]} if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then @@ -570,14 +570,14 @@ STATUS_NODE(){ fi if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running" + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running" touch ${NM_TMPFOLDER}/${nip}~${srvc}.down # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} [ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} & elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" if ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime touch ${NM_TMPFOLDER}/${nip}~${srvc}.down @@ -585,10 +585,10 @@ STATUS_NODE(){ touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent fi elif [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Yellow]}Disabled - Skipping" + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Yellow]}Disabled - Skipping" fi else - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) @@ -623,14 +623,14 @@ STATUS_NODE(){ ######################## ## DOCKER CHECK ######################## - if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" != "services" ]; then - if [ "${NMG_ACTION}" != "check" ]; then - [ "${NMG_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo + if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + [ "${STATUS_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}" DIVIDER . green 52 fi for docker in "${!dockers}"; do - if [ "${NMG_ACTION}" != "report" ]; then + if [ "${STATUS_ACTION}" != "report" ]; then c=0; cw=${PRI_CW}; spc='' spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done @@ -649,7 +649,7 @@ STATUS_NODE(){ SDCHK=true fi if [ ${SDCHK} != false ]; then - [ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " + [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then @@ -662,7 +662,7 @@ STATUS_NODE(){ fi sleep 10s if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en "${idsCL[Green]}Running - Fixed" fi SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed" @@ -670,7 +670,7 @@ STATUS_NODE(){ echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} else - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en "${idsCL[Red]}Not Running - Could Not Fix!" if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then @@ -684,7 +684,7 @@ STATUS_NODE(){ mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime fi toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)) - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi touch ${NM_TMPFOLDER}/${nip}~${docker}.down @@ -697,7 +697,7 @@ STATUS_NODE(){ echo -en "${idsCL[Yellow]}Disabled - Skipping" fi else - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -en "${idsCL[LightGreen]}Running" fi if [[ "${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}" = *"${docker}"* ]]; then @@ -716,11 +716,11 @@ STATUS_NODE(){ fi else - [ "${NMG_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}" + [ "${STATUS_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}" rm -f ${NM_TMPFOLDER}/${nip}~* if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then touch ${NM_TMPFOLDER}/${nip}.down - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then if [ "${checkhost}" != "false" ]; then echo -e "${idsCL[Red]} - Node is online, but SSH is down!${idsCL[Default]}" else @@ -745,7 +745,7 @@ STATUS_NODE(){ [ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=${errtime} fi - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then if [ "${checkhost}" != "false" ]; then echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" else @@ -764,7 +764,7 @@ STATUS_NODE(){ fi fi - [ "${NMG_ACTION}" != "check" ] && echo + [ "${STATUS_ACTION}" != "check" ] && echo done fi #GOFORCHECK diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 8fb99d56..e9c3a3a3 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # NodeMgmt - Node Management Scripts -ACTION="${1}" +NMG_ACTION="${1}" source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc @@ -46,7 +46,7 @@ BACKUP(){ echo DIVIDER echo -e "${idsCL[Green]}Backups are completed\n\n" - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then ENTER2CONTINUE fi } @@ -180,7 +180,7 @@ SET-PERMISSIONS(){ fi done if [ "${NEW_SITE}" != "" ]; then echo ; fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then ENTER2CONTINUE fi fi @@ -606,7 +606,7 @@ DOWNLIST(){ DOWNLIST=${DOWN_LIST} fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true fi echo @@ -727,7 +727,7 @@ DOWNLIST(){ echo fi - if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then + if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi @@ -738,11 +738,11 @@ MAINTENANCE_MODE(){ if [ "${NM_NODETYPES[${1^^}]}" != "" ]; then ntss=${1^^} ntypesel=(${ntss}) - NMG_ACTION=${2} + STATUS_ACTION=${2} else ntss="" ntypesel=(${NM_NODE_TYPES[@]}) - [ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1} + [ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1} fi PRI_CW=40 @@ -755,7 +755,7 @@ MAINTENANCE_MODE(){ NTHOSTS=${NODETYPE}_HOSTS[@] - if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${NMG_ACTION}" == "services" ]); then + if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then GOFORCHECK=false; else GOFORCHECK=true; @@ -763,9 +763,9 @@ MAINTENANCE_MODE(){ if [ ${GOFORCHECK} = true ]; then - if [ "${NMG_ACTION}" != "check" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER - if [ "${NMG_ACTION}" == "dockers" ]; then + if [ "${STATUS_ACTION}" == "dockers" ]; then echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}" else echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" @@ -780,7 +780,7 @@ MAINTENANCE_MODE(){ ######################## ## SERVICES CHECK ######################## - if [ "${NMG_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then + if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}" DIVIDER . green 52 @@ -791,7 +791,8 @@ MAINTENANCE_MODE(){ spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: " - echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}" + + echo -e "${idsCL[LightGreen]}here${idsCL[Default]}" done fi @@ -799,9 +800,9 @@ MAINTENANCE_MODE(){ ######################## ## DOCKER CHECK ######################## - if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" != "services" ]; then - if [ "${NMG_ACTION}" != "check" ]; then - [ "${NMG_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo + if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then + if [ "${STATUS_ACTION}" != "check" ]; then + [ "${STATUS_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}" DIVIDER . green 52 fi @@ -810,7 +811,8 @@ MAINTENANCE_MODE(){ spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " - echo -en "${idsCL[LightGreen]}Running${idsCL[Default]}" + + echo -e "${idsCL[LightGreen]}here${idsCL[Default]}" done fi @@ -918,7 +920,7 @@ GUI(){ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [[ "${noheader}" != *" ${3} "* ]] && [ "${2}" != "q" ] && [ "${3}" != "q" ]; then DISP_HEADER fi - case ${ACTION} in + case ${NMG_ACTION} in newcert) NEWCERT ${2} ${3};; certrenew) CERTRENEW;; delcert) DEL-SSL ${2};; @@ -1019,7 +1021,7 @@ GUI(){ ${NM_SCRIPT} status ${2} ${3} ${4} exit 0 else - [ "${NMG_ACTION}" != "report" ] && echo -e "${idsCL[LightRed]}Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!${idsCL[Default]}\n" + [ "${STATUS_ACTION}" != "report" ] && echo -e "${idsCL[LightRed]}Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!${idsCL[Default]}\n" if [ ! -f ${NM_TMPFOLDER}/.overload ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.overload)) -ge 600 ]; then SENDNOTICE "NODEMGMT OVERLOAD" "Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!" 1 touch ${NM_TMPFOLDER}/.overload @@ -1029,7 +1031,6 @@ GUI(){ fi fi;; scantimes) STATUS_SCANTIMES ${2} ${3} ${4} ${5};; - checkdocker) CHECK_DOCKER_SERVICES ${2};; set-permissions) SET-PERMISSIONS ${2};; service) if [[ ( "${2}" = "giteaxx" ) || ( "${2}" = "powerdns-adminxx" ) ]] && [ "${3}" = "start" ]; then