diff --git a/defaults.inc b/defaults.inc index 79283c8e..cba4b325 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='5.3.17-12012024' +VERS='5.3.17a-12012024' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' diff --git a/inc/status.inc b/inc/status.inc index 06ad2ed3..672ff2a9 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -261,7 +261,8 @@ STATUS(){ if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down else - if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([[ "${nip}" != *"10.2."* ]] || [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ]) && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then + # if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([[ "${nip}" != *"10.2."* ]] || [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ]) && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)) -gt 60 ]); then [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down @@ -626,7 +627,8 @@ STATUS_NODE(){ else [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})" - if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then + #if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime touch ${NM_TMPFOLDER}/${nip}~${srvc}.down if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] || [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then @@ -706,7 +708,8 @@ STATUS_NODE(){ [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: " if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + #if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ]; then if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then touch ${NM_TMPFOLDER}/${nip}~${docker}.down @@ -786,7 +789,8 @@ STATUS_NODE(){ [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.down ] && touch ${STATUSRUN_TMPFOLDER}/.${nip}.down rm -f ${NM_TMPFOLDER}/${nip}~* if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + # if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ]; then touch ${NM_TMPFOLDER}/${nip}.down fi if [ "${STATUS_ACTION}" != "check" ]; then @@ -799,7 +803,8 @@ STATUS_NODE(){ # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 [ "${checkhostssl}" != "ok" ] && echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} else - if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then + #if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)) -gt 60 ]; then mv ${NM_TMPFOLDER}/${nip}.down ${NM_TMPFOLDER}/${nip}.errtime toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) errtime=${toterrtime} @@ -821,7 +826,8 @@ STATUS_NODE(){ echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}" fi fi - if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then + # if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([ "${nip}" = "10.2.1.5" ] || [[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then + if [ ${NM_ENABLE_RENOTIFY} -eq 1 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && [ -f ${NM_TMPFOLDER}/${nip}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]; then if [ "${checkhost}" != "false" ]; then SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] SSH has been down for $(SHOW_TIME ${toterrtime})" 1 else @@ -829,7 +835,8 @@ STATUS_NODE(){ fi touch ${NM_TMPFOLDER}/${nip}.sent fi - if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + #if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && ([[ "${nip}" != *"10.2."* ]] || ([[ "${nip}" = *"10.2."* ]] && [ ! -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])); then + if [ ! -f ${NM_TMPFOLDER}/${nip}.disable ]; then touch ${NM_TMPFOLDER}/${nip}.down fi fi diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 3bd7f0af..8cb4b49f 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -918,7 +918,8 @@ DOWNLIST(){ fi [ ${is} -lt 10 ] && ignsel=" ${is}) " || ignsel="${is}) " - ([ -f ${NM_TMPFOLDER}/${ditem}.disable ] || ([[ "${host}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="OFF" || dstatus="${idsCL[Green]}ON" + # ([ -f ${NM_TMPFOLDER}/${ditem}.disable ] || ([[ "${host}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="OFF" || dstatus="${idsCL[Green]}ON" + [ -f ${NM_TMPFOLDER}/${ditem}.disable ] && dstatus="OFF" || dstatus="${idsCL[Green]}ON" c=0; spc3=''; until [ $c = `expr 26 - ${#time}` ]; do spc3="${spc3} "; ((c++)); done echo -e "\033[K${idsCL[LightRed]}${idsST[Bold]}${ignsel}${idsST[Reset]}${idsCL[LightRed]}${itemtype}${spca}${itemname}${spc}${host}${spc2}${time}${spc3}${dstatus}${idsCL[Default]}" fi @@ -992,7 +993,8 @@ MAINTENANCE_MODE(){ if [ "${choice}" -gt 0 ] 2>/dev/null && [ ${item_number} -eq ${choice} ]; then [ ! -f ${NM_TMPFOLDER}/${nip}.disable ] && touch ${NM_TMPFOLDER}/${nip}.disable || rm -f ${NM_TMPFOLDER}/${nip}.disable fi - ([ -f ${NM_TMPFOLDER}/${nip}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + # ([ -f ${NM_TMPFOLDER}/${nip}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + [ -f ${NM_TMPFOLDER}/${nip}.disable ] && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" echo -e "${dstatus}${idsCL[Default]}" ((item_number++)) @@ -1014,7 +1016,8 @@ MAINTENANCE_MODE(){ if [ "${choice}" -gt 0 ] 2>/dev/null && [ ${item_number} -eq ${choice} ]; then [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && touch ${NM_TMPFOLDER}/${nip}~${srvc}.disable || rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable fi - ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + # ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" echo -e "${dstatus}${idsCL[Default]}" ((item_number++)) @@ -1038,7 +1041,8 @@ MAINTENANCE_MODE(){ if [ "${choice}" -gt 0 ] 2>/dev/null && [ ${item_number} -eq ${choice} ]; then [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && touch ${NM_TMPFOLDER}/${nip}~${docker}.disable || rm -f ${NM_TMPFOLDER}/${nip}~${docker}.disable fi - ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + # ([ -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + [ -f ${NM_TMPFOLDER}/${nip}~${docker}.disable ] && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" echo -e "${dstatus}${idsCL[Default]}" ((item_number++)) @@ -1072,7 +1076,8 @@ MAINTENANCE_MODE(){ if [ "${choice}" -gt 0 ] 2>/dev/null && [ ${item_number} -eq ${choice} ]; then [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable || rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable fi - ([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + #([ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] || ([[ "${nip}" = *"10.2."* ]] && [ -f ${PW_TMPFOLDER}/OFF-vMS-Host1.down ])) && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" + [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.disable ] && dstatus="${idsCL[LightRed]}OFF" || dstatus="${idsCL[Green]}ON" echo -e "${dstatus}${idsCL[Default]}" ((item_number++))