diff --git a/defaults.inc b/defaults.inc index a002cc5b..055e4cd9 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,24 +1,24 @@ -VERS='4.8.4-07032023' +VERS='4.8.6-07042023' noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns ' CERT_DAEMON='/snap/bin/certbot' -FOLDER=/opt/idssys/nodemgmt -SCRIPT=${FOLDER}/nodemgmt-scripts.sh -TMPFOLDER=${FOLDER}/.tmp -LOGFOLDER=${FOLDER}/logs -LOGFILE=${LOGFOLDER}/logfile +NM_FOLDER=/opt/idssys/nodemgmt +NM_SCRIPT=${NM_FOLDER}/nodemgmt-scripts.sh +NM_TMPFOLDER=${NM_FOLDER}/.tmp +NM_LOGFOLDER=${NM_FOLDER}/logs +NM_LOGFILE=${NM_LOGFOLDER}/logfile RENOTIFY=1800 -[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER} -[ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER} +[ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER} +[ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER} declare -i errtime NM_SRVCOPT=(start stop restart reload enable disable) -if [ -f ${FOLDER}/defaults.local.inc ]; then +if [ -f ${NM_FOLDER}/defaults.local.inc ]; then declare -A NM_HOSTS declare -A NM_SERVICES_CHECK declare -A NM_DOCKERS_CHECK @@ -35,7 +35,7 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then declare -A NM_SINGLESRVR_DOCKERS declare -A NM_SINGLESRVR_IP - source ${FOLDER}/defaults.local.inc + source ${NM_FOLDER}/defaults.local.inc IFS=, LOCAL_SERVICES=(${LOCAL_SERVICES}) @@ -133,7 +133,7 @@ NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban GET-CHECKCERT-DOMAINS(){ declare -A CHECKCERT_DOMAINS IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " diff --git a/inc/certs.inc b/inc/certs.inc index 6c94117c..e661a32b 100755 --- a/inc/certs.inc +++ b/inc/certs.inc @@ -38,10 +38,10 @@ NEWCERT(){ touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then - rm -f ${FOLDER}/cert-request.lastrun + rm -f ${NM_FOLDER}/cert-request.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun - yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun + echo -e "${NEW_CERT}\n${daterun}" > ${NM_FOLDER}/cert-request.lastrun + yes | cp -rfH ${NM_FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun DIVIDER true if [ "${2}" != "newsite" ]; then echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}" @@ -143,13 +143,13 @@ CERTRENEW(){ echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" echo sleep 5 - $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - # $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun - # $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun + $CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun + # $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun + # $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun CONCAT_SSL - chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun - yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun + chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun + yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun DIVIDER true @@ -157,7 +157,7 @@ CERTRENEW(){ ssh root@webserver-node2.scity.us service nginx start echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" echo - SERVICE nginx restart 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun + SERVICE nginx restart 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}" echo if [ -z $action ] || [ "${action}" = "gui" ]; then @@ -166,18 +166,18 @@ CERTRENEW(){ fi } NIGHTLYRENEW(){ - rm -f ${FOLDER}/cert-renewal.lastrun + rm -f ${NM_FOLDER}/cert-renewal.lastrun ssh root@webserver-node2.scity.us service nginx stop sleep 5 - $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun + $CERT_DAEMON renew --webroot -w /var/www/html &>> ${NM_FOLDER}/cert-renewal.lastrun CONCAT_SSL - chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun - chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun - yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun + chown -R root:letsencrypt /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun + chmod -R 6775 /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun + yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun ssh root@webserver-node2.scity.us service nginx start - SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun + SERVICE nginx reload web &>> ${NM_FOLDER}/cert-renewal.lastrun } CONCAT_SSL(){ @@ -193,7 +193,7 @@ LISTCERTS(){ declare -i cw; declare -i spc1; declare -i c declare -A CHECKCERT_DOMAINS IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " @@ -272,7 +272,7 @@ LISTCERTS_NPM(){ declare -i cw; declare -i spc1; declare -i c declare -A CHECKCERT_DOMAINS IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " @@ -405,17 +405,17 @@ CHECK_NPMCERTS(){ CHECK-CERTS(){ if [ "${1}" == "check" ]; then - rm -f ${FOLDER}/ssl-cert-check.lastrun - ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun - SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)" + rm -f ${NM_FOLDER}/ssl-cert-check.lastrun + ${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains > ${NM_FOLDER}/ssl-cert-check.lastrun + SSLLOG="$(cat ${NM_FOLDER}/ssl-cert-check.lastrun)" SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG" else - # ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains + # ${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains IFS=$'\n' - for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } - CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N) + CERTINFO=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N) CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}') CERTVALID=${CERTVALID: -1} CERTDAYS=${CERTINFO#*days=} diff --git a/inc/sites.inc b/inc/sites.inc index 731b07d5..db139077 100755 --- a/inc/sites.inc +++ b/inc/sites.inc @@ -404,7 +404,7 @@ NEWSITE(){ ######################################### PROXY else - cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig} + cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig} sed -i "s/<>/${NGINX_SERVERNAME}/g" ${nginxconfig} sed -i "s/<>/${MAIN_SITE}/g" ${nginxconfig} sed -i "s/<>/${PROXYHOST}/g" ${nginxconfig} @@ -423,11 +423,11 @@ NEWSITE(){ # fi fi - rm -f ${FOLDER}/new-site.lastrun + rm -f ${NM_FOLDER}/new-site.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` - echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun - # yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun - # yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun + echo -e "${NEW_SITE}\n${daterun}" > ${NM_FOLDER}/new-site.lastrun + # yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun + # yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /var/www/new-site.lastrun # daterun=`date +%Y-%m-%d-%H-%M-%S` # echo -e "${daterun}" >> /etc/nginx/new-site.lastrun DIVIDER true @@ -464,7 +464,7 @@ NEWSITE(){ fi else - ${SCRIPT} newsite + ${NM_SCRIPT} newsite exit 0 fi } @@ -488,7 +488,7 @@ NEWPROXYSITE_CREATE(){ # NGINX_SERVERNAME=${SITENAME} # fi - # cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig} + # cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig} # sed -i "s/<>/${NGINX_SERVERNAME}/g" ${nginxconfig} # sed -i "s/<>/${MAIN_SITE}/g" ${nginxconfig} # sed -i "s/<>/${PROXYHOST}/g" ${nginxconfig} diff --git a/inc/status.inc b/inc/status.inc index 357074c1..5b43c574 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -18,13 +18,13 @@ STATUS(){ fi for NTYPE in "${NODE_TYPES[@]}"; do PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then + if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do # echo "CHECK: ${NTYPE} - ${!PH} - $rcheck - ${NM_REPL_CHECK_LOC[${rcheck}]}" - ${PH_CMD} rm -f ${FOLDER}/test.repl + ${PH_CMD} rm -f ${NM_FOLDER}/test.repl daterun=`date +%Y-%m-%d-%H-%M-%S` ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" & done @@ -73,39 +73,39 @@ STATUS(){ fi if [ "$(systemctl is-active ${srvc})" != "active" ]; then - if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then + if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} + touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then - mv ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime + if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then + mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime fi - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi - touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down + touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down fi fi else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" fi - if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then - if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime) + if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then + if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) fi - rm -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.* + rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.* SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${LOGFILE} + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE} fi fi @@ -161,13 +161,13 @@ STATUS(){ DIVIDER false green fi - if [ -f ${TMPFOLDER}/${nip}.down ]; then - if [ -f ${TMPFOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) + if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then + if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) fi - rm -f ${TMPFOLDER}/${nip}.* + rm -f ${NM_TMPFOLDER}/${nip}.* SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" fi @@ -197,21 +197,21 @@ STATUS(){ fi # echo "mysqlgo=${mysqlgo}" if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" - touch ${TMPFOLDER}/${nip}~${srvc}.down + touch ${NM_TMPFOLDER}/${nip}~${srvc}.down # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then - [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) - touch ${TMPFOLDER}/${nip}~${srvc}.down + [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) + touch ${NM_TMPFOLDER}/${nip}~${srvc}.down SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 # ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} & - elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then + elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 fi @@ -219,17 +219,17 @@ STATUS(){ fi else [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}" - if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then - [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) + if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then + [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} - rm -f ${TMPFOLDER}/${nip}~${srvc}.* + rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* fi # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then if [ "${srvc}" == "gitea" ]; then - rm -f ${FOLDER}/*~${srvc}.* + rm -f ${NM_FOLDER}/*~${srvc}.* fi fi [ "${mysqlgo}" = "true" ] && unset mysqlgo @@ -267,22 +267,22 @@ STATUS(){ # fi # # echo "mysqlgo=${mysqlgo}" # if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then - # if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then + # if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" # fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - # touch ${TMPFOLDER}/${nip}~${srvc}.down - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE} + # touch ${NM_TMPFOLDER}/${nip}~${srvc}.down + # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} # # else - # errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) + # errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) # if [ $errtime -gt ${RENOTIFY} ]; then - # if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then - # mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime + # if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then + # mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime # fi - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) - # touch ${TMPFOLDER}/${nip}~${srvc}.down + # toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) + # touch ${NM_TMPFOLDER}/${nip}~${srvc}.down # if [ "${srvc}" = "mysql" ]; then # ${NCMD} systemctl restart ${srvc} # fi @@ -295,23 +295,23 @@ STATUS(){ # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Green]}Running${idsCL[Default]}" # fi - # if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then - # if [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) + # if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then + # if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then + # toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) # else - # toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down) + # toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) # fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE} + # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} # - # rm -f ${TMPFOLDER}/${nip}~${srvc}.down - # rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime + # rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down + # rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime # # fi # # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then # if [ "${srvc}" == "gitea" ]; then - # rm -f ${FOLDER}/*~${srvc}.down - # rm -f ${FOLDER}/*~${srvc}.errtime + # rm -f ${NM_FOLDER}/*~${srvc}.down + # rm -f ${NM_FOLDER}/*~${srvc}.errtime # fi # fi # [ "${mysqlgo}" = "true" ] && unset mysqlgo @@ -346,10 +346,10 @@ STATUS(){ if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} else ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} fi sleep 10s if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then @@ -357,28 +357,28 @@ STATUS(){ echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" fi SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" - rm -f ${TMPFOLDER}/${nip}~${docker}.* - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE} + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 - touch ${TMPFOLDER}/${nip}~${docker}.down - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE} + touch ${NM_TMPFOLDER}/${nip}~${docker}.down + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down) + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down) if [ $errtime -gt ${RENOTIFY} ]; then - if [ ! -f ${TMPFOLDER}/${nip}~${docker}.errtime ]; then - mv ${TMPFOLDER}/${nip}~${docker}.down ${TMPFOLDER}/${nip}~${docker}.errtime + if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then + mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime fi - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi - touch ${TMPFOLDER}/${nip}~${docker}.down + touch ${NM_TMPFOLDER}/${nip}~${docker}.down fi fi @@ -388,34 +388,34 @@ STATUS(){ if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" fi - rm -f ${TMPFOLDER}/${nip}~${docker}.* + rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* fi fi done fi else - if [ ! -f ${TMPFOLDER}/${nip}.down ]; then - touch ${TMPFOLDER}/${nip}.down - if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then - touch ${TMPFOLDER}/${nip}.errtime + if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then + touch ${NM_TMPFOLDER}/${nip}.down + if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + touch ${NM_TMPFOLDER}/${nip}.errtime fi if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi if [ $errtime -gt ${RENOTIFY} ]; then - touch ${TMPFOLDER}/${nip}.down + touch ${NM_TMPFOLDER}/${nip}.down SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1 - elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then + elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 fi # if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then @@ -452,7 +452,7 @@ STATUS(){ var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then + if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1 @@ -476,7 +476,7 @@ STATUS(){ echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi - if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then + if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do if [ "${ST_ACTION}" != "check" ]; then @@ -491,34 +491,34 @@ STATUS(){ checked="" until [ "${checked}" != "" ]; do - if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then + if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then checked=good - elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then + elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then checked=timeout fi - rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.* + rm -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.* done if [ "${checked}" == "timeout" ]; then [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" - if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then - touch ${TMPFOLDER}/${nip}~${rcheck}.down + if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then + touch ${NM_TMPFOLDER}/${nip}~${rcheck}.down SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} fi else [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" - if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then - rm -f ${TMPFOLDER}/${nip}~${rcheck}.down + if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then + rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} + echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE} fi fi done else if [ "${ST_ACTION}" != "check" ]; then - [ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}" - [ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}" + [ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}" + [ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}" fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi @@ -538,25 +538,25 @@ STATUS(){ #WHCMD="ssh root@${WEB_HOSTS[0]}" for NTYPE in "${NODE_TYPES[@]}"; do PH=${NTYPE}_HOSTS[0] - if [ ! -f ${FOLDER}/${!PH}.down ]; then + if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do - ${PH_CMD} rm -f ${FOLDER}/test.repl + ${PH_CMD} rm -f ${NM_FOLDER}/test.repl daterun=`date +%Y-%m-%d-%H-%M-%S` if [ "${PH_CMD}" = "" ]; then - rm -f ${FOLDER}/test.repl & + rm -f ${NM_FOLDER}/test.repl & rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & else - ${PH_CMD} rm -f ${FOLDER}/test.repl & + ${PH_CMD} rm -f ${NM_FOLDER}/test.repl & ${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl & fi done fi fi done - rm -Rf ${FOLDER}/test.repl + rm -Rf ${NM_FOLDER}/test.repl if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo @@ -709,5 +709,5 @@ REPLCHECK(){ fi [ "`date +%s`" -gt "${timeout}" ] && timeout=true && break done - [ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good + [ "${timeout}" == "true" ] && touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good } \ No newline at end of file diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 5c5057be..7987d1c2 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -40,14 +40,14 @@ CHECK_DOCKER_SERVICES(){ echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green fi - if [ -f ${TMPFOLDER}/${nip}.down ]; then - if [ -f ${TMPFOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) + if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then + if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) fi - rm -f ${TMPFOLDER}/${nip}.down - rm -f ${TMPFOLDER}/${nip}.errtime + rm -f ${NM_TMPFOLDER}/${nip}.down + rm -f ${NM_TMPFOLDER}/${nip}.errtime SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" fi @@ -80,16 +80,16 @@ CHECK_DOCKER_SERVICES(){ fi done else - if [ ! -f ${TMPFOLDER}/${nip}.down ]; then - touch ${TMPFOLDER}/${nip}.down - if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then - touch ${TMPFOLDER}/${nip}.errtime + if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then + touch ${NM_TMPFOLDER}/${nip}.down + if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + touch ${NM_TMPFOLDER}/${nip}.errtime fi else - errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down) + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) fi if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi fi @@ -122,7 +122,7 @@ BACKUP(){ echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}" fi done - touch ${FOLDER}/backup.lastrun + touch ${NM_FOLDER}/backup.lastrun # if [ "${action}" != "backup" ]; then echo DIVIDER true @@ -602,7 +602,7 @@ HAPROXY-CONFIG() { ${NCMD} "nodemgmt update q" ${NCMD} "rm -f /etc/haproxy/haproxy.cfg" - ${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" + ${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` @@ -623,7 +623,7 @@ HAPROXY-CONFIG() { ${NCMD} "nodemgmt update q" ${NCMD} "rm -f /etc/haproxy/haproxy.cfg" - ${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" + ${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` @@ -643,7 +643,7 @@ HAPROXY-CONFIG() { ${NCMD} "nodemgmt update q" ${NCMD} "rm -f /etc/haproxy/haproxy.cfg" - ${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" + ${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` @@ -672,7 +672,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ for stc in "${!servicestocheck}"; do if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" - echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE} + echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE} [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" for nip in "${!hosts}"; do @@ -712,7 +712,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ fi if [ "$(systemctl is-active nginx)" != "active" ]; then - ${SCRIPT} service web restart ${local_ip} + ${NM_SCRIPT} service web restart ${local_ip} fi fi } @@ -734,7 +734,7 @@ KEEPALIVE-CONFIG() { ${NCMD} "nodemgmt update q" ${NCMD} "rm -f /etc/keepalived/keepalived.conf" - ${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf" + ${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` @@ -755,7 +755,7 @@ KEEPALIVE-CONFIG() { ${NCMD} "nodemgmt update q" ${NCMD} "rm -f /etc/keepalived/keepalived.conf" - ${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf" + ${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` @@ -778,15 +778,15 @@ NODEUPDATE() { declare -i cw; declare -i spc1; declare -i c echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}" echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" - nip=${NM_NODEMANAGER} - if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then - nip='localhost' + nmip=${NM_NODEMANAGER} + if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nmip}"* ]]; then + nmip='localhost' fi c=0; cw=24; spca=''; spc1=${cw}-7; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done - echo -en "Updating Node-Manager[$nip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " - ssh -tq root@${nip} "nodemgmt update q" + echo -en "Updating Node-Manager[$nmip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : " + ssh -tq root@${nmip} "nodemgmt update q" echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : " - ssh -tq root@${nip} "runup runup" + ssh -tq root@${nmip} "runup runup" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" echo for NTYPE in "${NODE_TYPES[@]}"; do @@ -803,6 +803,8 @@ NODEUPDATE() { ${NCMD} "nodemgmt update q" >/dev/null 2>&1 echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : " ${NCMD} "runup runup" >/dev/null 2>&1 + echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}Sync Defaults${idsCL[Default]} : " + ssh -tq root@${nmip} "scp ${NM_FOLDER}/defaults.local.inc root@${nip}:${NM_FOLDER}/" echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}" nid=`expr $nid + 1` done @@ -833,7 +835,7 @@ VCENTER-SSL(){ CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}" - VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N) + VCCERTDAYS=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N) VCCERTDAYS=${VCCERTDAYS#*=} if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then @@ -899,10 +901,10 @@ VCENTER-SSL(){ echo } ADD_LOGROTATE_CRONTAB(){ - if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then - (crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab - + if ! crontab -l | grep -q "${NM_FOLDER}/tmp-logrotate"; then + (crontab -l ; echo "0 */1 * * * logrotate -f ${NM_FOLDER}/tmp-logrotate")| crontab - fi - /bin/chmod 0644 ${FOLDER}/tmp-logrotate + /bin/chmod 0644 ${NM_FOLDER}/tmp-logrotate echo echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}" echo @@ -1005,7 +1007,7 @@ DOWNLIST(){ idsST=('') fi declare -i cw; declare -i spc1; declare -i c - DOWN_LIST=`find ${FOLDER} -name "*.down"` + DOWN_LIST=`find ${NM_FOLDER} -name "*.down"` if [ -z $action ] || [ "${action}" = "gui" ]; then DIVIDER true @@ -1038,10 +1040,10 @@ DOWNLIST(){ [ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}" fi - if [ -f ${TMPFOLDER}/${ditem}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.errtime) + if [ -f ${NM_TMPFOLDER}/${ditem}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime) else - toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.down) + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down) fi c=0; spc='' @@ -1182,8 +1184,8 @@ GUI(){ update) ;; update-nodes) if [ "${2}" == "report" ]; then - rm -f ${LOGFOLDER}/update-nodes.lastrun - NODEUPDATE ${2} > ${LOGFOLDER}/update-nodes.lastrun + rm -f ${NM_LOGFOLDER}/update-nodes.lastrun + NODEUPDATE ${2} > ${NM_LOGFOLDER}/update-nodes.lastrun else NODEUPDATE ${2} fi @@ -1192,44 +1194,44 @@ GUI(){ backup) BACKUP;; downlist) if [ "${2}" = "report" ]; then - if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then - rm -f ${LOGFOLDER}/status-downlist.lastrun - DOWNLIST report >> ${LOGFOLDER}/status-downlist.lastrun - STATUSLOG="$(cat ${LOGFOLDER}/status-downlist.lastrun)" + if [ "$(find ${NM_FOLDER} -name '*.down')" != "" ]; then + rm -f ${NM_LOGFOLDER}/status-downlist.lastrun + DOWNLIST report >> ${NM_LOGFOLDER}/status-downlist.lastrun + STATUSLOG="$(cat ${NM_LOGFOLDER}/status-downlist.lastrun)" SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}" fi else DOWNLIST ${2} fi;; status) - if [ -f "${FOLDER}/status-check.running" ]; then - fd=`stat --format=%Y "${FOLDER}/status-check.running"` + if [ -f "${NM_FOLDER}/status-check.running" ]; then + fd=`stat --format=%Y "${NM_FOLDER}/status-check.running"` cbd=$((`date +%s` - 450)) - [ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running" + [ ${fd} -lt ${cbd} ] && rm -f "${NM_FOLDER}/status-check.running" fi - if [ ! -f "${FOLDER}/status-check.running" ]; then + if [ ! -f "${NM_FOLDER}/status-check.running" ]; then start_status=`date +%s` - touch "${FOLDER}/status-check.running" + touch "${NM_FOLDER}/status-check.running" if [ "${2}" == "report" ]; then - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${LOGFILE} - echo -e "$(date) - Starting status check #############################\n" > ${FOLDER}/status-check.running - STATUS report ${3} >> ${FOLDER}/status-check.running - echo -e "\n\n$(date) - Finished status check #############################\n" >> ${FOLDER}/status-check.running - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${LOGFILE} - rm -f ${FOLDER}/status-check.lastrun - mv ${FOLDER}/status-check.running ${FOLDER}/status-check.lastrun + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} + echo -e "$(date) - Starting status check #############################\n" > ${NM_FOLDER}/status-check.running + STATUS report ${3} >> ${NM_FOLDER}/status-check.running + echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_FOLDER}/status-check.running + echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} + rm -f ${NM_FOLDER}/status-check.lastrun + mv ${NM_FOLDER}/status-check.running ${NM_FOLDER}/status-check.lastrun end_status=`date +%s` runtime_status=$((end_status-start_status)) - echo -e "\nRuntime: ${runtime_status}" >> ${FOLDER}/status-check.lastrun + echo -e "\nRuntime: ${runtime_status}" >> ${NM_FOLDER}/status-check.lastrun if [ "${3}" = "email" ]; then - echo -e "\n\n" >> ${FOLDER}/status-check.lastrun - DOWNLIST report >> ${FOLDER}/status-check.lastrun - STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)" + echo -e "\n\n" >> ${NM_FOLDER}/status-check.lastrun + DOWNLIST report >> ${NM_FOLDER}/status-check.lastrun + STATUSLOG="$(cat ${NM_FOLDER}/status-check.lastrun)" SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}" fi else STATUS ${2} ${3} - rm -f ${FOLDER}/status-check.running + rm -f ${NM_FOLDER}/status-check.running fi else # echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}" @@ -1237,8 +1239,8 @@ GUI(){ echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}" read -n1 choice case $choice in - [Ll]) watch -n1 tail -n35 ${FOLDER}/status-check.running; echo;; - [Pp]) echo; cat ${FOLDER}/status-check.lastrun;; + [Ll]) watch -n1 tail -n35 ${NM_FOLDER}/status-check.running; echo;; + [Pp]) echo; cat ${NM_FOLDER}/status-check.lastrun;; esac echo fi;; diff --git a/run.sh b/run.sh index 584ae44a..97f46f37 100755 --- a/run.sh +++ b/run.sh @@ -35,7 +35,7 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ " git pull >/dev/null 2>&1 git submodule update --remote >/dev/null 2>&1 fi - cd ${FOLDER} + cd ${NM_FOLDER} if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then # if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then if [ "${2}" != "q" ]; then @@ -52,7 +52,7 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ " git pull >/dev/null 2>&1 git submodule update --remote >/dev/null 2>&1 /bin/chmod +x /opt/idssys/nodemgmt/nodemgmt-scripts.sh 2>&1 - source ${FOLDER}/defaults.inc + source ${NM_FOLDER}/defaults.inc if [ "${2}" != "q" ]; then echo -en "\e[1A"; echo -e "\e[0K\r ${idsCL[Green]}Update to v${VERS} complete${idsCL[Default]}" @@ -68,5 +68,5 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ " echo -e "${idsCL[Red]}Could not connect to 'git.schroedercity.com' for updates${idsCL[Default]}" fi fi -${SCRIPT} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25} +${NM_SCRIPT} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25} exit 0