voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-04 11:09:40 -05:00
parent f53920a042
commit 3f8e796124
6 changed files with 199 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
inc/certs.inc
View File

@@ -38,10 +38,10 @@ NEWCERT(){
touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then
rm -f ${FOLDER}/cert-request.lastrun
rm -f ${NM_FOLDER}/cert-request.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
echo -e "${NEW_CERT}\n${daterun}" > ${NM_FOLDER}/cert-request.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
DIVIDER true
if [ "${2}" != "newsite" ]; then
echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}"
@@ -143,13 +143,13 @@ CERTRENEW(){
echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
echo
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
DIVIDER true
@@ -157,7 +157,7 @@ CERTRENEW(){
ssh root@webserver-node2.scity.us service nginx start
echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
echo
SERVICE nginx restart 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
SERVICE nginx restart 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
echo
if [ -z $action ] || [ "${action}" = "gui" ]; then
@@ -166,18 +166,18 @@ CERTRENEW(){
fi
}
NIGHTLYRENEW(){
rm -f ${FOLDER}/cert-renewal.lastrun
rm -f ${NM_FOLDER}/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx stop
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
$CERT_DAEMON renew --webroot -w /var/www/html &>> ${NM_FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
chown -R root:letsencrypt /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx start
SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun
SERVICE nginx reload web &>> ${NM_FOLDER}/cert-renewal.lastrun
}
CONCAT_SSL(){
@@ -193,7 +193,7 @@ LISTCERTS(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
@@ -272,7 +272,7 @@ LISTCERTS_NPM(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
@@ -405,17 +405,17 @@ CHECK_NPMCERTS(){
CHECK-CERTS(){
if [ "${1}" == "check" ]; then
rm -f ${FOLDER}/ssl-cert-check.lastrun
${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
rm -f ${NM_FOLDER}/ssl-cert-check.lastrun
${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains > ${NM_FOLDER}/ssl-cert-check.lastrun
SSLLOG="$(cat ${NM_FOLDER}/ssl-cert-check.lastrun)"
SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
else
# ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains
# ${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N)
CERTINFO=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N)
CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}')
CERTVALID=${CERTVALID: -1}
CERTDAYS=${CERTINFO#*days=}

14
inc/sites.inc
View File

@@ -404,7 +404,7 @@ NEWSITE(){
######################################### PROXY
else
cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig}
cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig}
sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" ${nginxconfig}
sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" ${nginxconfig}
sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" ${nginxconfig}
@@ -423,11 +423,11 @@ NEWSITE(){
# fi
fi
rm -f ${FOLDER}/new-site.lastrun
rm -f ${NM_FOLDER}/new-site.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
echo -e "${NEW_SITE}\n${daterun}" > ${NM_FOLDER}/new-site.lastrun
# yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
# yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /var/www/new-site.lastrun
# daterun=`date +%Y-%m-%d-%H-%M-%S`
# echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
DIVIDER true
@@ -464,7 +464,7 @@ NEWSITE(){
fi
else
${SCRIPT} newsite
${NM_SCRIPT} newsite
exit 0
fi
}
@@ -488,7 +488,7 @@ NEWPROXYSITE_CREATE(){
# NGINX_SERVERNAME=${SITENAME}
# fi
# cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig}
# cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig}
# sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" ${nginxconfig}
# sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" ${nginxconfig}
# sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" ${nginxconfig}

186
inc/status.inc
View File

@@ -18,13 +18,13 @@ STATUS(){
fi
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - $rcheck - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done
@@ -73,39 +73,39 @@ STATUS(){
fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
fi
rm -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${LOGFILE}
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
fi
fi
@@ -161,13 +161,13 @@ STATUS(){
DIVIDER false green
fi
if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.*
rm -f ${NM_TMPFOLDER}/${nip}.*
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
@@ -197,21 +197,21 @@ STATUS(){
fi
# echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}~${srvc}.down
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${TMPFOLDER}/${nip}~${srvc}.down
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
fi
@@ -219,17 +219,17 @@ STATUS(){
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
rm -f ${TMPFOLDER}/${nip}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
fi
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
if [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*~${srvc}.*
rm -f ${NM_FOLDER}/*~${srvc}.*
fi
fi
[ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -267,22 +267,22 @@ STATUS(){
# fi
# # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
#
# else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
# fi
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc}
# fi
@@ -295,23 +295,23 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
#
# rm -f ${TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
#
# fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
# if [ "${srvc}" == "gitea" ]; then
# rm -f ${FOLDER}/*~${srvc}.down
# rm -f ${FOLDER}/*~${srvc}.errtime
# rm -f ${NM_FOLDER}/*~${srvc}.down
# rm -f ${NM_FOLDER}/*~${srvc}.errtime
# fi
# fi
# [ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -346,10 +346,10 @@ STATUS(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
@@ -357,28 +357,28 @@ STATUS(){
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${TMPFOLDER}/${nip}~${docker}.down ${TMPFOLDER}/${nip}~${docker}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${nip}~${docker}.down
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
fi
fi
@@ -388,34 +388,34 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${TMPFOLDER}/${nip}~${docker}.*
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
fi
fi
done
fi
else
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${TMPFOLDER}/${nip}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
if [ $errtime -gt ${RENOTIFY} ]; then
touch ${TMPFOLDER}/${nip}.down
touch ${NM_TMPFOLDER}/${nip}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
@@ -452,7 +452,7 @@ STATUS(){
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
@@ -476,7 +476,7 @@ STATUS(){
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
@@ -491,34 +491,34 @@ STATUS(){
checked=""
until [ "${checked}" != "" ]; do
if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
checked=good
elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
checked=timeout
fi
rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
rm -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.*
done
if [ "${checked}" == "timeout" ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -538,25 +538,25 @@ STATUS(){
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl &
rm -f ${NM_FOLDER}/test.repl &
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else
${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi
done
fi
fi
done
rm -Rf ${FOLDER}/test.repl
rm -Rf ${NM_FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
@@ -709,5 +709,5 @@ REPLCHECK(){
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done
[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
[ "${timeout}" == "true" ] && touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good
}