voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-04 11:09:40 -05:00
parent f53920a042
commit 3f8e796124
6 changed files with 199 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
defaults.inc
View File

@@ -1,24 +1,24 @@
VERS='4.8.4-07032023'
VERS='4.8.6-07042023'
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns '
CERT_DAEMON='/snap/bin/certbot'
FOLDER=/opt/idssys/nodemgmt
SCRIPT=${FOLDER}/nodemgmt-scripts.sh
TMPFOLDER=${FOLDER}/.tmp
LOGFOLDER=${FOLDER}/logs
LOGFILE=${LOGFOLDER}/logfile
NM_FOLDER=/opt/idssys/nodemgmt
NM_SCRIPT=${NM_FOLDER}/nodemgmt-scripts.sh
NM_TMPFOLDER=${NM_FOLDER}/.tmp
NM_LOGFOLDER=${NM_FOLDER}/logs
NM_LOGFILE=${NM_LOGFOLDER}/logfile
RENOTIFY=1800
[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
[ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER}
[ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER}
[ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER}
declare -i errtime
NM_SRVCOPT=(start stop restart reload enable disable)
if [ -f ${FOLDER}/defaults.local.inc ]; then
if [ -f ${NM_FOLDER}/defaults.local.inc ]; then
declare -A NM_HOSTS
declare -A NM_SERVICES_CHECK
declare -A NM_DOCKERS_CHECK
@@ -35,7 +35,7 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then
declare -A NM_SINGLESRVR_DOCKERS
declare -A NM_SINGLESRVR_IP
source ${FOLDER}/defaults.local.inc
source ${NM_FOLDER}/defaults.local.inc
IFS=,
LOCAL_SERVICES=(${LOCAL_SERVICES})
@@ -133,7 +133,7 @@ NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban
GET-CHECKCERT-DOMAINS(){
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "

48
inc/certs.inc
View File

@@ -38,10 +38,10 @@ NEWCERT(){
touch /opt/lb-data/letsencrypt/live/${MAIN_CERT}/newcert
if [ -f /opt/lb-data/nginx/sites-enabled/${MAIN_CERT}.conf ]; then
rm -f ${FOLDER}/cert-request.lastrun
rm -f ${NM_FOLDER}/cert-request.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
yes | cp -rfH ${FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
echo -e "${NEW_CERT}\n${daterun}" > ${NM_FOLDER}/cert-request.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-request.lastrun /opt/lb-data/letsencrypt/cert-request.lastrun
DIVIDER true
if [ "${2}" != "newsite" ]; then
echo -e -n "${idsCL[LightCyan]}Reload NGINX on LB Nodes (Y/n): ${idsCL[Default]}"
@@ -143,13 +143,13 @@ CERTRENEW(){
echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
echo
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${NM_FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
DIVIDER true
@@ -157,7 +157,7 @@ CERTRENEW(){
ssh root@webserver-node2.scity.us service nginx start
echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
echo
SERVICE nginx restart 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
SERVICE nginx restart 2>&1 | tee -a ${NM_FOLDER}/cert-renewal.lastrun
echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
echo
if [ -z $action ] || [ "${action}" = "gui" ]; then
@@ -166,18 +166,18 @@ CERTRENEW(){
fi
}
NIGHTLYRENEW(){
rm -f ${FOLDER}/cert-renewal.lastrun
rm -f ${NM_FOLDER}/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx stop
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
$CERT_DAEMON renew --webroot -w /var/www/html &>> ${NM_FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
chown -R root:letsencrypt /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt &>> ${NM_FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${NM_FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx start
SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun
SERVICE nginx reload web &>> ${NM_FOLDER}/cert-renewal.lastrun
}
CONCAT_SSL(){
@@ -193,7 +193,7 @@ LISTCERTS(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
@@ -272,7 +272,7 @@ LISTCERTS_NPM(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
@@ -405,17 +405,17 @@ CHECK_NPMCERTS(){
CHECK-CERTS(){
if [ "${1}" == "check" ]; then
rm -f ${FOLDER}/ssl-cert-check.lastrun
${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
rm -f ${NM_FOLDER}/ssl-cert-check.lastrun
${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains > ${NM_FOLDER}/ssl-cert-check.lastrun
SSLLOG="$(cat ${NM_FOLDER}/ssl-cert-check.lastrun)"
SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
else
# ${FOLDER}/ssl-cert-check/ssl-cert-check -f ${FOLDER}/ssl-cert-check/ssldomains
# ${NM_FOLDER}/ssl-cert-check/ssl-cert-check -f ${NM_FOLDER}/ssl-cert-check/ssldomains
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
CERTINFO=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N)
CERTINFO=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p ${PORT} -s ${HOST} -N)
CERTVALID=$(echo ${CERTINFO} | awk -F' valid' '{print $1}')
CERTVALID=${CERTVALID: -1}
CERTDAYS=${CERTINFO#*days=}

14
inc/sites.inc
View File

@@ -404,7 +404,7 @@ NEWSITE(){
######################################### PROXY
else
cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig}
cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig}
sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" ${nginxconfig}
sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" ${nginxconfig}
sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" ${nginxconfig}
@@ -423,11 +423,11 @@ NEWSITE(){
# fi
fi
rm -f ${FOLDER}/new-site.lastrun
rm -f ${NM_FOLDER}/new-site.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
echo -e "${NEW_SITE}\n${daterun}" > ${NM_FOLDER}/new-site.lastrun
# yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
# yes | cp -rfH ${NM_FOLDER}/new-site.lastrun /var/www/new-site.lastrun
# daterun=`date +%Y-%m-%d-%H-%M-%S`
# echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
DIVIDER true
@@ -464,7 +464,7 @@ NEWSITE(){
fi
else
${SCRIPT} newsite
${NM_SCRIPT} newsite
exit 0
fi
}
@@ -488,7 +488,7 @@ NEWPROXYSITE_CREATE(){
# NGINX_SERVERNAME=${SITENAME}
# fi
# cp ${FOLDER}/templates/nginx.proxy.site ${nginxconfig}
# cp ${NM_FOLDER}/templates/nginx.proxy.site ${nginxconfig}
# sed -i "s/<<SERVER_NAME>>/${NGINX_SERVERNAME}/g" ${nginxconfig}
# sed -i "s/<<MAIN_SITE>>/${MAIN_SITE}/g" ${nginxconfig}
# sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" ${nginxconfig}

186
inc/status.inc
View File

@@ -18,13 +18,13 @@ STATUS(){
fi
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - $rcheck - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done
@@ -73,39 +73,39 @@ STATUS(){
fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
fi
rm -f ${TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${LOGFILE}
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
fi
fi
@@ -161,13 +161,13 @@ STATUS(){
DIVIDER false green
fi
if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.*
rm -f ${NM_TMPFOLDER}/${nip}.*
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
@@ -197,21 +197,21 @@ STATUS(){
fi
# echo "mysqlgo=${mysqlgo}"
if [ "${srvctst[$sr]}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${TMPFOLDER}/${nip}~${srvc}.down
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
[ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${TMPFOLDER}/${nip}~${srvc}.down
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# ([ "${srvc}" == "mysql" ] || [ "${srvc}" == "mariadb" ]) && ${NCMD} systemctl restart ${srvc} &
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
fi
@@ -219,17 +219,17 @@ STATUS(){
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
rm -f ${TMPFOLDER}/${nip}~${srvc}.*
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
fi
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
if [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*~${srvc}.*
rm -f ${NM_FOLDER}/*~${srvc}.*
fi
fi
[ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -267,22 +267,22 @@ STATUS(){
# fi
# # echo "mysqlgo=${mysqlgo}"
# if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${LOGFILE}
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
#
# else
# errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# if [ $errtime -gt ${RENOTIFY} ]; then
# if [ ! -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${TMPFOLDER}/${nip}~${srvc}.down ${TMPFOLDER}/${nip}~${srvc}.errtime
# if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
# fi
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${TMPFOLDER}/${nip}~${srvc}.down
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# if [ "${srvc}" = "mysql" ]; then
# ${NCMD} systemctl restart ${srvc}
# fi
@@ -295,23 +295,23 @@ STATUS(){
# if [ "${ST_ACTION}" != "check" ]; then
# echo -e "${idsCL[Green]}Running${idsCL[Default]}"
# fi
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.errtime)
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
# if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ]; then
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)
# else
# toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${srvc}.down)
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
#
# rm -f ${TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${TMPFOLDER}/${nip}~${srvc}.errtime
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
#
# fi
# # if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
# if [ "${srvc}" == "gitea" ]; then
# rm -f ${FOLDER}/*~${srvc}.down
# rm -f ${FOLDER}/*~${srvc}.errtime
# rm -f ${NM_FOLDER}/*~${srvc}.down
# rm -f ${NM_FOLDER}/*~${srvc}.errtime
# fi
# fi
# [ "${mysqlgo}" = "true" ] && unset mysqlgo
@@ -346,10 +346,10 @@ STATUS(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
@@ -357,28 +357,28 @@ STATUS(){
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${TMPFOLDER}/${nip}~${docker}.down ${TMPFOLDER}/${nip}~${docker}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.errtime ]; then
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${TMPFOLDER}/${nip}~${docker}.down
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
fi
fi
@@ -388,34 +388,34 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${TMPFOLDER}/${nip}~${docker}.*
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
fi
fi
done
fi
else
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${TMPFOLDER}/${nip}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
if [ $errtime -gt ${RENOTIFY} ]; then
touch ${TMPFOLDER}/${nip}.down
touch ${NM_TMPFOLDER}/${nip}.down
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] has been down for $(SHOW_TIME ${toterrtime})" 1
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
elif [ $errtime -gt 60 ] && [ $errtime -lt 180 ] && [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
fi
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
@@ -452,7 +452,7 @@ STATUS(){
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
@@ -476,7 +476,7 @@ STATUS(){
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]} (${!PH}) <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
if [ ! -f ${FOLDER}/${!PH}.down ] && [ ! -f ${TMPFOLDER}/${nip}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
@@ -491,34 +491,34 @@ STATUS(){
checked=""
until [ "${checked}" != "" ]; do
if [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
if [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good ]; then
checked=good
elif [ -f ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
elif [ -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout ]; then
checked=timeout
fi
rm -f ${TMPFOLDER}/repl.${rcheck}.${nip}.*
rm -f ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.*
done
if [ "${checked}" == "timeout" ]; then
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
[ -f ${FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
@@ -538,25 +538,25 @@ STATUS(){
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
if [ ! -f ${FOLDER}/${!PH}.down ]; then
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${NM_FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl &
rm -f ${NM_FOLDER}/test.repl &
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else
${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_FOLDER}/test.repl &
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi
done
fi
fi
done
rm -Rf ${FOLDER}/test.repl
rm -Rf ${NM_FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
@@ -709,5 +709,5 @@ REPLCHECK(){
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done
[ "${timeout}" == "true" ] && touch ${TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${TMPFOLDER}/repl.${rcheck}.${nip}.good
[ "${timeout}" == "true" ] && touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.timeout || touch ${NM_TMPFOLDER}/repl.${rcheck}.${nip}.good
}

120
nodemgmt-scripts.sh
View File

@@ -40,14 +40,14 @@ CHECK_DOCKER_SERVICES(){
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.down
rm -f ${TMPFOLDER}/${nip}.errtime
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
@@ -80,16 +80,16 @@ CHECK_DOCKER_SERVICES(){
fi
done
else
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${TMPFOLDER}/${nip}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
@@ -122,7 +122,7 @@ BACKUP(){
echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
fi
done
touch ${FOLDER}/backup.lastrun
touch ${NM_FOLDER}/backup.lastrun
# if [ "${action}" != "backup" ]; then
echo
DIVIDER true
@@ -602,7 +602,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -623,7 +623,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -643,7 +643,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -672,7 +672,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
for nip in "${!hosts}"; do
@@ -712,7 +712,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
fi
if [ "$(systemctl is-active nginx)" != "active" ]; then
${SCRIPT} service web restart ${local_ip}
${NM_SCRIPT} service web restart ${local_ip}
fi
fi
}
@@ -734,7 +734,7 @@ KEEPALIVE-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -755,7 +755,7 @@ KEEPALIVE-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -778,15 +778,15 @@ NODEUPDATE() {
declare -i cw; declare -i spc1; declare -i c
echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
nip=${NM_NODEMANAGER}
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
nip='localhost'
nmip=${NM_NODEMANAGER}
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nmip}"* ]]; then
nmip='localhost'
fi
c=0; cw=24; spca=''; spc1=${cw}-7; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
echo -en "Updating Node-Manager[$nip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
ssh -tq root@${nip} "nodemgmt update q"
echo -en "Updating Node-Manager[$nmip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
ssh -tq root@${nmip} "nodemgmt update q"
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
ssh -tq root@${nip} "runup runup"
ssh -tq root@${nmip} "runup runup"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
echo
for NTYPE in "${NODE_TYPES[@]}"; do
@@ -803,6 +803,8 @@ NODEUPDATE() {
${NCMD} "nodemgmt update q" >/dev/null 2>&1
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
${NCMD} "runup runup" >/dev/null 2>&1
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}Sync Defaults${idsCL[Default]} : "
ssh -tq root@${nmip} "scp ${NM_FOLDER}/defaults.local.inc root@${nip}:${NM_FOLDER}/"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
@@ -833,7 +835,7 @@ VCENTER-SSL(){
CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
VCCERTDAYS=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
VCCERTDAYS=${VCCERTDAYS#*=}
if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
@@ -899,10 +901,10 @@ VCENTER-SSL(){
echo
}
ADD_LOGROTATE_CRONTAB(){
if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then
(crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab -
if ! crontab -l | grep -q "${NM_FOLDER}/tmp-logrotate"; then
(crontab -l ; echo "0 */1 * * * logrotate -f ${NM_FOLDER}/tmp-logrotate")| crontab -
fi
/bin/chmod 0644 ${FOLDER}/tmp-logrotate
/bin/chmod 0644 ${NM_FOLDER}/tmp-logrotate
echo
echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
echo
@@ -1005,7 +1007,7 @@ DOWNLIST(){
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
DOWN_LIST=`find ${FOLDER} -name "*.down"`
DOWN_LIST=`find ${NM_FOLDER} -name "*.down"`
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
@@ -1038,10 +1040,10 @@ DOWNLIST(){
[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"
fi
if [ -f ${TMPFOLDER}/${ditem}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.errtime)
if [ -f ${NM_TMPFOLDER}/${ditem}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down)
fi
c=0; spc=''
@@ -1182,8 +1184,8 @@ GUI(){
update) ;;
update-nodes)
if [ "${2}" == "report" ]; then
rm -f ${LOGFOLDER}/update-nodes.lastrun
NODEUPDATE ${2} > ${LOGFOLDER}/update-nodes.lastrun
rm -f ${NM_LOGFOLDER}/update-nodes.lastrun
NODEUPDATE ${2} > ${NM_LOGFOLDER}/update-nodes.lastrun
else
NODEUPDATE ${2}
fi
@@ -1192,44 +1194,44 @@ GUI(){
backup) BACKUP;;
downlist)
if [ "${2}" = "report" ]; then
if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then
rm -f ${LOGFOLDER}/status-downlist.lastrun
DOWNLIST report >> ${LOGFOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${LOGFOLDER}/status-downlist.lastrun)"
if [ "$(find ${NM_FOLDER} -name '*.down')" != "" ]; then
rm -f ${NM_LOGFOLDER}/status-downlist.lastrun
DOWNLIST report >> ${NM_LOGFOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${NM_LOGFOLDER}/status-downlist.lastrun)"
SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
fi
else
DOWNLIST ${2}
fi;;
status)
if [ -f "${FOLDER}/status-check.running" ]; then
fd=`stat --format=%Y "${FOLDER}/status-check.running"`
if [ -f "${NM_FOLDER}/status-check.running" ]; then
fd=`stat --format=%Y "${NM_FOLDER}/status-check.running"`
cbd=$((`date +%s` - 450))
[ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running"
[ ${fd} -lt ${cbd} ] && rm -f "${NM_FOLDER}/status-check.running"
fi
if [ ! -f "${FOLDER}/status-check.running" ]; then
if [ ! -f "${NM_FOLDER}/status-check.running" ]; then
start_status=`date +%s`
touch "${FOLDER}/status-check.running"
touch "${NM_FOLDER}/status-check.running"
if [ "${2}" == "report" ]; then
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${LOGFILE}
echo -e "$(date) - Starting status check #############################\n" > ${FOLDER}/status-check.running
STATUS report ${3} >> ${FOLDER}/status-check.running
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${FOLDER}/status-check.running
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${LOGFILE}
rm -f ${FOLDER}/status-check.lastrun
mv ${FOLDER}/status-check.running ${FOLDER}/status-check.lastrun
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE}
echo -e "$(date) - Starting status check #############################\n" > ${NM_FOLDER}/status-check.running
STATUS report ${3} >> ${NM_FOLDER}/status-check.running
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_FOLDER}/status-check.running
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE}
rm -f ${NM_FOLDER}/status-check.lastrun
mv ${NM_FOLDER}/status-check.running ${NM_FOLDER}/status-check.lastrun
end_status=`date +%s`
runtime_status=$((end_status-start_status))
echo -e "\nRuntime: ${runtime_status}" >> ${FOLDER}/status-check.lastrun
echo -e "\nRuntime: ${runtime_status}" >> ${NM_FOLDER}/status-check.lastrun
if [ "${3}" = "email" ]; then
echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
DOWNLIST report >> ${FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
echo -e "\n\n" >> ${NM_FOLDER}/status-check.lastrun
DOWNLIST report >> ${NM_FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${NM_FOLDER}/status-check.lastrun)"
SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
fi
else
STATUS ${2} ${3}
rm -f ${FOLDER}/status-check.running
rm -f ${NM_FOLDER}/status-check.running
fi
else
# echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
@@ -1237,8 +1239,8 @@ GUI(){
echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}"
read -n1 choice
case $choice in
[Ll]) watch -n1 tail -n35 ${FOLDER}/status-check.running; echo;;
[Pp]) echo; cat ${FOLDER}/status-check.lastrun;;
[Ll]) watch -n1 tail -n35 ${NM_FOLDER}/status-check.running; echo;;
[Pp]) echo; cat ${NM_FOLDER}/status-check.lastrun;;
esac
echo
fi;;

6
run.sh
View File

@@ -35,7 +35,7 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "
git pull >/dev/null 2>&1
git submodule update --remote >/dev/null 2>&1
fi
cd ${FOLDER}
cd ${NM_FOLDER}
if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
# if [ "`git log --pretty=%H ...refs/heads/dev^ | head -n 1`" != "`git ls-remote origin -h refs/heads/dev |cut -f1`" ]; then
if [ "${2}" != "q" ]; then
@@ -52,7 +52,7 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "
git pull >/dev/null 2>&1
git submodule update --remote >/dev/null 2>&1
/bin/chmod +x /opt/idssys/nodemgmt/nodemgmt-scripts.sh 2>&1
source ${FOLDER}/defaults.inc
source ${NM_FOLDER}/defaults.inc
if [ "${2}" != "q" ]; then
echo -en "\e[1A";
echo -e "\e[0K\r ${idsCL[Green]}Update to v${VERS} complete${idsCL[Default]}"
@@ -68,5 +68,5 @@ if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "
echo -e "${idsCL[Red]}Could not connect to 'git.schroedercity.com' for updates${idsCL[Default]}"
fi
fi
${SCRIPT} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25}
${NM_SCRIPT} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25}
exit 0