voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-11-23 23:16:36 -06:00
parent 12fb442f77
commit 3d85e36794
5 changed files with 40 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
VERS='4.15.15-11232023'
VERS='4.15.20-11232023'
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log '
CERT_DAEMON='/snap/bin/certbot'

28
inc/certs.inc
View File

@@ -679,29 +679,37 @@ export PDNS_Ttl=60
scp -q /tmp/vcenter-update-ssl.sh root@${NM_VC_HOSTNAME}:/tmp/vcenter-update-ssl.sh
ssh -q root@${NM_VC_HOSTNAME} "bash /tmp/vcenter-update-ssl.sh"
LIVEMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum /etc/vmware-rhttpproxy/ssl/rui.crt | cut -d\ -f1")
CURRENTMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum ${VC_CERT} | cut -d\ -f1")
if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}"
echo
if [ ! -f ${NM_VC_ACMEFOLDER}/${NM_VC_HOSTNAME}/${NM_VC_HOSTNAME}.cer ]; then
echo -e "${idsCL[LightYellow]}No certificate was created${idsCL[Default]}\n"
exit 0
elif [ $(expr `date +%s` - $(stat -c %Y ${NM_VC_ACMEFOLDER}/${NM_VC_HOSTNAME}/${NM_VC_HOSTNAME}.cer)) -ge 600 ]; then
echo -e "${idsCL[LightYellow]}The certificate was unable to be updated${idsCL[Default]}\n"
exit 0
else
LIVEMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum /etc/vmware-rhttpproxy/ssl/rui.crt | cut -d\ -f1")
CURRENTMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum ${VC_CERT} | cut -d\ -f1")
if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
echo -e "${idsCL[LightYellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}\n"
exit 0
fi
fi
echo -e "${idsCL[LightGreen]}Updating certificates on vCenter... ${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}"
echo
ssh -q root@${NM_VC_HOSTNAME} "(printf '1\n%s\n' '${NM_VC_USER}'; sleep 1; printf '%s\n' '${NM_VC_PASS}'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '${VC_CERT}' '${VC_KEY}' '${VC_CHAIN}') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
# ssh -q root@${NM_VC_HOSTNAME} "(printf '1\n%s\n' '${NM_VC_USER}'; sleep 1; printf '%s\n' '${NM_VC_PASS}'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '${VC_CERT}' '${VC_KEY}' '${VC_CHAIN}') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
SENDNOTICE "vCenter SSL Updated" "Refresh/rescan any systems connecting to vcenter like Veeam"
SENDNOTICE "vCenter SSL Updated" "Refresh/Reconnect any systems connecting to vCenter, like Veeam Dummy!"
echo "$(date +%Y-%m-%d-%H-%M-%S) VCENTER SSL HAS BEEN UPDATED" >> ${NM_LOGFOLDER}/vc-ssl.log
ssh -q root@${NM_VC_HOSTNAME} "rm -f /tmp/vcenter-update-ssl.sh"
rm -f /tmp/vcenter-update-ssl.sh
echo -e "\n${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}\n"
echo -e "${idsCL[LightCyan]}Don't forget to redo the vCenter connection in Veeam${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}vCenter info not configured in 'defaults.local.inc'${idsCL[Default]}\n"
echo -e "${idsCL[LightYellow]}vCenter info not configured in 'defaults.local.inc'${idsCL[Default]}\n"
fi
}

8
inc/services.inc
View File

@@ -22,7 +22,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
if [ "${gofor}" == "true" ]; then
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
echo "$(date) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${stc} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${stc} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket"
for nip in "${!hosts}"; do
@@ -35,7 +35,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
[ "${stc}" = "headscale" ] && sleep 90s
/bin/systemctl start ${stc} &
echo "$(date) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE}
fi
done
for docker in "${!dockerstocheck}"; do
@@ -46,7 +46,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
[ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1
elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then
/usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
@@ -57,7 +57,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
[ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=auth_redis)" ] && ssh root@${nip} /usr/bin/docker stop auth_redis >/dev/null 2>&1
elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then
ssh root@${nip} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
fi

28
inc/status.inc
View File

@@ -92,7 +92,7 @@ STATUS(){
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down)
if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then
@@ -118,7 +118,7 @@ STATUS(){
fi
rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.*
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE}
fi
fi
@@ -236,7 +236,7 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
@@ -259,7 +259,7 @@ STATUS(){
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.*
@@ -307,7 +307,7 @@ STATUS(){
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
# touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
#
# else
# errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
@@ -336,7 +336,7 @@ STATUS(){
# toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)
# fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
# echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
# echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
#
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down
# rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
@@ -383,10 +383,10 @@ STATUS(){
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE}
else
${NCMD} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
@@ -395,7 +395,7 @@ STATUS(){
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${NM_TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
@@ -404,7 +404,7 @@ STATUS(){
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then
@@ -443,7 +443,7 @@ STATUS(){
echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE}
else
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
@@ -544,7 +544,7 @@ STATUS(){
mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE}
else
snderrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)`
# echo "HERE: $snderrtime"
@@ -552,7 +552,7 @@ STATUS(){
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)`
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It has been down for $(SHOW_TIME ${toterrtime})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
fi
fi
@@ -564,7 +564,7 @@ It has been down for $(SHOW_TIME ${toterrtime})" 1
toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)`
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE}
fi
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
fi

6
nodemgmt-scripts.sh
View File

@@ -843,11 +843,11 @@ GUI(){
touch "${NM_TMPFOLDER}/status-check.running"
if [ "${2}" == "report" ]; then
# start_status=`date +%s`
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE}
echo -e "$(date) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE}
echo -e "$(date +%Y-%m-%d-%H-%M-%S) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running
STATUS report ${3} >> ${NM_TMPFOLDER}/status-check.running
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_TMPFOLDER}/status-check.running
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE}
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE}
rm -f ${NM_LOGFOLDER}/status-check.lastrun
mv ${NM_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun