voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-12 09:07:27 -06:00
parent 9cd897f175
commit e05c11473f
5 changed files with 106 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
VERS='5.1.12-12122023'
VERS='5.1.13-12122023'
NM_BETA=false
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck '

16
inc/certs.inc
View File

@@ -86,7 +86,7 @@ NEWCERT(){
fi
echo
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -145,7 +145,7 @@ CERTRENEW(){
SERVICE_MGMT nginx reload >> ${NM_LOGFOLDER}/cert-renewal.lastrun
echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
echo
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -205,7 +205,7 @@ LISTCERTS(){
rm -f /tmp/ssllist
# #for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done > /tmp/ssllist
# fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
fi
echo -e "\033[K${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
@@ -441,7 +441,7 @@ LISTCERTS(){
echo
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -468,7 +468,7 @@ LISTCERTS_NPM(){
rm -f /tmp/ssllist
for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done > /tmp/ssllist
fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
fi
echo
@@ -516,7 +516,7 @@ LISTCERTS_NPM(){
DIVIDER false darkGray 120
done
echo
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -539,7 +539,7 @@ COPYCERTS_NPM(){
done
fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
fi
@@ -552,7 +552,7 @@ COPYCERTS_NPM(){
fi
done
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
echo
DIVIDER true
ENTER2CONTINUE

87
inc/services.inc
View File

@@ -445,93 +445,6 @@ SERVICE_MGMT(){
fi
}
CHECK_DOCKER_SERVICES(){
for NTYPE in "${NM_NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
if [ "${NMG_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
for nip in "${!hosts}"; do
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
uptime=`${NCMD} uptime -p`
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
else
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
fi
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
for docker in "${!dockers}"; do
if [ "${NMG_ACTION}" != "check" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker}
fi
${NCMD} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
else
if [ "${NMG_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
fi
fi
else
if [ "${NMG_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
fi
done
else
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
fi
if [ "${NMG_ACTION}" != "check" ]; then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
fi
fi
if [ "${NMG_ACTION}" != "check" ]; then echo; fi
done
fi
done
}

154
inc/status.inc
View File

@@ -4,16 +4,16 @@ STATUS(){
log_start=$(date "+%Y-%m-%d %H:%M:%S")
# if [ "${1}" == "report" ] || [ "${1}" == "" ]; then
# NMG_ACTION=${1}
# STATUS_ACTION=${1}
# ntypesel=(${NM_NODE_TYPES[@]})
# # if [ "${NM_NODETYPES[${NMG_ACTION^^}]}" != "" ]; then
# # ntss=${NMG_ACTION^^}
# # if [ "${NM_NODETYPES[${STATUS_ACTION^^}]}" != "" ]; then
# # ntss=${STATUS_ACTION^^}
# # ntypesel=(${ntss})
# # NMG_ACTION=${2}
# # STATUS_ACTION=${2}
# # else
# # ntss=""
# # ntypesel=(${NM_NODE_TYPES[@]})
# # [ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1}
# # [ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1}
# # fi
# else
# while [ $# -gt 0 ]; do
@@ -38,11 +38,11 @@ STATUS(){
if [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
ntss=${1^^}
ntypesel=(${ntss})
NMG_ACTION=${2}
STATUS_ACTION=${2}
else
ntss=""
ntypesel=(${NM_NODE_TYPES[@]})
[ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1}
[ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1}
fi
PRI_CW=40
@@ -55,15 +55,15 @@ STATUS(){
############################
([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge 300 ]) && REPLRUN=1 || REPLRUN=0
# if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then
if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then
if [ "${NMG_ACTION}" != "check" ]; then
# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
fi
for NTYPE in "${ntypesel[@]}"; do
REPLSTART ${NTYPE} &
done
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
echo -en "${idsCL[LightCyan]}Starting processes to collect/monitor replication check : "
fi
@@ -78,38 +78,38 @@ STATUS(){
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${NMG_ACTION}" & >/dev/null 2>&1
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${STATUS_ACTION}" & >/dev/null 2>&1
done
fi
fi
done
fi
done
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
fi
if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "services" ] || [ "${NMG_ACTION}" == "dockers" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "all" ]; then
if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "services" ] || [ "${STATUS_ACTION}" == "dockers" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "all" ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : "
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : "
for NTYPE in "${ntypesel[@]}"; do
STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running 2>/dev/null &
done
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
########################
## LOCALHOST CHECK
########################
if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${NMG_ACTION}" != "dockers" ] && [ "${NMG_ACTION}" != "services" ] && [ "${ntss}" == "" ]; then
if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${STATUS_ACTION}" != "dockers" ] && [ "${STATUS_ACTION}" != "services" ] && [ "${ntss}" == "" ]; then
cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
lip=${RUN_NODE_IP}
echo -e "\n$(DIVIDER)\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}"
echo -e "$(DIVIDER)"
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} ${idsST[Reset]}${idsCL[LightCyan]}[${lip}]"
if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then
if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
uptime=`uptime -p`
echo -e "${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
else
@@ -118,15 +118,15 @@ STATUS(){
DIVIDER . green
fi
# if [ "${NMG_ACTION}" != "check" ]; then
# if [ "${STATUS_ACTION}" != "check" ]; then
# lip=${RUN_NODE_IP}
# uptime=`uptime -p`
# echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
# DIVIDER false green
# fi
for srvc in "${LOCAL_SERVICES[@]}"; do
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${NMG_ACTION}" != "report" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
@@ -138,7 +138,7 @@ STATUS(){
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
@@ -147,12 +147,12 @@ STATUS(){
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
[ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi
fi
else
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
@@ -204,9 +204,9 @@ STATUS(){
## REPLICATION CHECK
########################
# if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then
if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
if [ "${NMG_ACTION}" != "check" ]; then
# if [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "check" ] || [ "${STATUS_ACTION}" == "" ]; then
if [ "${STATUS_ACTION}" == "repl" ] || [ "${STATUS_ACTION}" == "" ] || ([ "${STATUS_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -en "${idsST[Reset]}"
@@ -220,15 +220,15 @@ STATUS(){
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${NMG_ACTION}" != "report" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
@@ -247,7 +247,7 @@ STATUS(){
done
if [ "${checked}" == "timeout" ]; then
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
else
@@ -262,7 +262,7 @@ It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDE
fi
elif [ "${checked}" == "good" ]; then
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
@@ -276,12 +276,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
done
else
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
[ "${NMG_ACTION}" != "check" ] && echo
[ "${STATUS_ACTION}" != "check" ] && echo
fi
@@ -291,7 +291,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
##########################
# REMOVE REPL CHECK FILES
##########################
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
for NTYPE in "${ntypesel[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
@@ -312,16 +312,16 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
done
rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl
touch ${NM_TMPFOLDER}/.replcheck
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
fi
########################
## FREE SPACE CHECK
########################
#if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "freespace" ]; then
if [ "${NMG_ACTION}" == "freespace" ]; then
#if [ "${STATUS_ACTION}" == "" ] || [ "${STATUS_ACTION}" == "report" ] || [ "${STATUS_ACTION}" == "freespace" ]; then
if [ "${STATUS_ACTION}" == "freespace" ]; then
for NTYPE in "${ntypesel[@]}"; do
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
@@ -331,12 +331,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
fi
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
[ "${NMG_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
declare -A partitions
DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
@@ -352,7 +352,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
pname=${pname#*vg-}
if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
freespace=${DRIVEINFO[$((${ii}+2))]//M/}
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
c=0; cw=20; spc=''
spc1=${cw}-${#pname}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
@@ -376,7 +376,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
fs_status=''
fs_status_color='Green'
fi
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
fi
done
@@ -384,7 +384,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
# if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
# pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
# pname=${pname#*vg-}
# if [ "${NMG_ACTION}" != "check" ]; then
# if [ "${STATUS_ACTION}" != "check" ]; then
# c=0; cw=20; spc=''
# spc1=${cw}-${#pname}
# until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
@@ -409,7 +409,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
# fs_status_color='Green'
#
# fi
# if [ "${NMG_ACTION}" != "check" ]; then
# if [ "${STATUS_ACTION}" != "check" ]; then
# if [[ ${freespace} -gt 1000 ]]; then
# fsgb=$(bc <<< "scale=2; ${freespace}/1000")
# fsdsp="${fsgb} GB"
@@ -427,9 +427,9 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
done
fi
########################
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo
if [ -z ${ACTION} ] || [ "${ACTION}" == "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" == "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -437,7 +437,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
end=`date +%s`
runtime=$((end-start))
echo -e "Runtime: ${runtime}\n"
[ "${NMG_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes
[ "${STATUS_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes
}
STATUS_NODE(){
@@ -447,7 +447,7 @@ STATUS_NODE(){
NTHOSTS=${NODETYPE}_HOSTS[@]
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${NMG_ACTION}" == "services" ]); then
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then
GOFORCHECK=false;
else
GOFORCHECK=true;
@@ -455,9 +455,9 @@ STATUS_NODE(){
if [ ${GOFORCHECK} = true ]; then
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
if [ "${NMG_ACTION}" == "dockers" ]; then
if [ "${STATUS_ACTION}" == "dockers" ]; then
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
else
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
@@ -465,9 +465,9 @@ STATUS_NODE(){
DIVIDER . lightYellow; echo -en "${idsST[Reset]}"
fi
for nip in "${!NTHOSTS}"; do
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
[ "${NMG_ACTION}" != "report" ] && echo -en " - Verifying Host..."
[ "${STATUS_ACTION}" != "report" ] && echo -en " - Verifying Host..."
fi
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
@@ -486,8 +486,8 @@ STATUS_NODE(){
[ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
if [ "${NMG_ACTION}" != "check" ]; then
if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
if ([ "${STATUS_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${STATUS_ACTION}" != "report" ]; then
if [ "${NODETYPE}" == "OFW" ]; then
uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
if [ ${#uptime} -ge 6 ]; then
@@ -501,7 +501,7 @@ STATUS_NODE(){
else
uptime=$(${NCMD} uptime -p)
fi
[ "${NMG_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
[ "${STATUS_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
else
echo -e "${idsCL[Default]}"
fi
@@ -520,7 +520,7 @@ STATUS_NODE(){
########################
## SERVICES CHECK
########################
if [ "${NMG_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
DIVIDER . green 52
@@ -529,7 +529,7 @@ STATUS_NODE(){
[ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`)
sr=0
for srvc in "${!srvcs}"; do
if [ "${NMG_ACTION}" != "report" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
@@ -551,7 +551,7 @@ STATUS_NODE(){
[ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false
if [ ${SSCHK} != false ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
if [ "${NODETYPE}" != "OFW" ]; then
srvctest=${srvctst[$sr]}
if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
@@ -570,14 +570,14 @@ STATUS_NODE(){
fi
if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
if ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
@@ -585,10 +585,10 @@ STATUS_NODE(){
touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
fi
elif [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Yellow]}Disabled - Skipping"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Yellow]}Disabled - Skipping"
fi
else
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
@@ -623,14 +623,14 @@ STATUS_NODE(){
########################
## DOCKER CHECK
########################
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" != "services" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
[ "${NMG_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
[ "${STATUS_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
DIVIDER . green 52
fi
for docker in "${!dockers}"; do
if [ "${NMG_ACTION}" != "report" ]; then
if [ "${STATUS_ACTION}" != "report" ]; then
c=0; cw=${PRI_CW}; spc=''
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
@@ -649,7 +649,7 @@ STATUS_NODE(){
SDCHK=true
fi
if [ ${SDCHK} != false ]; then
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then
@@ -662,7 +662,7 @@ STATUS_NODE(){
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Green]}Running - Fixed"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${docker}' ${NM_DOCKER_DESC[${docker}]} - is fixed"
@@ -670,7 +670,7 @@ STATUS_NODE(){
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
@@ -684,7 +684,7 @@ STATUS_NODE(){
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
@@ -697,7 +697,7 @@ STATUS_NODE(){
echo -en "${idsCL[Yellow]}Disabled - Skipping"
fi
else
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -en "${idsCL[LightGreen]}Running"
fi
if [[ "${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}" = *"${docker}"* ]]; then
@@ -716,11 +716,11 @@ STATUS_NODE(){
fi
else
[ "${NMG_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
[ "${STATUS_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
rm -f ${NM_TMPFOLDER}/${nip}~*
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[Red]} - Node is online, but SSH is down!${idsCL[Default]}"
else
@@ -745,7 +745,7 @@ STATUS_NODE(){
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=${errtime}
fi
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
if [ "${checkhost}" != "false" ]; then
echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
else
@@ -764,7 +764,7 @@ STATUS_NODE(){
fi
fi
[ "${NMG_ACTION}" != "check" ] && echo
[ "${STATUS_ACTION}" != "check" ] && echo
done
fi #GOFORCHECK

39
nodemgmt-scripts.sh
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env bash
# NodeMgmt - Node Management Scripts
ACTION="${1}"
NMG_ACTION="${1}"
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
@@ -46,7 +46,7 @@ BACKUP(){
echo
DIVIDER
echo -e "${idsCL[Green]}Backups are completed\n\n"
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
ENTER2CONTINUE
fi
}
@@ -180,7 +180,7 @@ SET-PERMISSIONS(){
fi
done
if [ "${NEW_SITE}" != "" ]; then echo ; fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
ENTER2CONTINUE
fi
fi
@@ -606,7 +606,7 @@ DOWNLIST(){
DOWNLIST=${DOWN_LIST}
fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
fi
echo
@@ -727,7 +727,7 @@ DOWNLIST(){
echo
fi
if [ -z ${ACTION} ] || [ "${ACTION}" = "gui" ]; then
if [ -z ${NMG_ACTION} ] || [ "${NMG_ACTION}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
@@ -738,11 +738,11 @@ MAINTENANCE_MODE(){
if [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
ntss=${1^^}
ntypesel=(${ntss})
NMG_ACTION=${2}
STATUS_ACTION=${2}
else
ntss=""
ntypesel=(${NM_NODE_TYPES[@]})
[ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1}
[ "${1}" == "sync" ] && STATUS_ACTION=repl || STATUS_ACTION=${1}
fi
PRI_CW=40
@@ -755,7 +755,7 @@ MAINTENANCE_MODE(){
NTHOSTS=${NODETYPE}_HOSTS[@]
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${NMG_ACTION}" == "services" ]); then
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${STATUS_ACTION}" == "services" ]); then
GOFORCHECK=false;
else
GOFORCHECK=true;
@@ -763,9 +763,9 @@ MAINTENANCE_MODE(){
if [ ${GOFORCHECK} = true ]; then
if [ "${NMG_ACTION}" != "check" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
if [ "${NMG_ACTION}" == "dockers" ]; then
if [ "${STATUS_ACTION}" == "dockers" ]; then
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
else
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
@@ -780,7 +780,7 @@ MAINTENANCE_MODE(){
########################
## SERVICES CHECK
########################
if [ "${NMG_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
if [ "${STATUS_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
DIVIDER . green 52
@@ -791,7 +791,8 @@ MAINTENANCE_MODE(){
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}here${idsCL[Default]}"
done
fi
@@ -799,9 +800,9 @@ MAINTENANCE_MODE(){
########################
## DOCKER CHECK
########################
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" != "services" ]; then
if [ "${NMG_ACTION}" != "check" ]; then
[ "${NMG_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${STATUS_ACTION}" != "services" ]; then
if [ "${STATUS_ACTION}" != "check" ]; then
[ "${STATUS_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
DIVIDER . green 52
fi
@@ -810,7 +811,8 @@ MAINTENANCE_MODE(){
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
echo -en "${idsCL[LightGreen]}Running${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}here${idsCL[Default]}"
done
fi
@@ -918,7 +920,7 @@ GUI(){
if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [[ "${noheader}" != *" ${3} "* ]] && [ "${2}" != "q" ] && [ "${3}" != "q" ]; then
DISP_HEADER
fi
case ${ACTION} in
case ${NMG_ACTION} in
newcert) NEWCERT ${2} ${3};;
certrenew) CERTRENEW;;
delcert) DEL-SSL ${2};;
@@ -1019,7 +1021,7 @@ GUI(){
${NM_SCRIPT} status ${2} ${3} ${4}
exit 0
else
[ "${NMG_ACTION}" != "report" ] && echo -e "${idsCL[LightRed]}Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!${idsCL[Default]}\n"
[ "${STATUS_ACTION}" != "report" ] && echo -e "${idsCL[LightRed]}Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!${idsCL[Default]}\n"
if [ ! -f ${NM_TMPFOLDER}/.overload ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.overload)) -ge 600 ]; then
SENDNOTICE "NODEMGMT OVERLOAD" "Whooooaaaaa there cowboy, we got 3 threads running already.... easy killer!" 1
touch ${NM_TMPFOLDER}/.overload
@@ -1029,7 +1031,6 @@ GUI(){
fi
fi;;
scantimes) STATUS_SCANTIMES ${2} ${3} ${4} ${5};;
checkdocker) CHECK_DOCKER_SERVICES ${2};;
set-permissions) SET-PERMISSIONS ${2};;
service)
if [[ ( "${2}" = "giteaxx" ) || ( "${2}" = "powerdns-adminxx" ) ]] && [ "${3}" = "start" ]; then