From 3d85e36794e2f01e85f905ac8552bf01c34329f5 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Thu, 23 Nov 2023 23:16:36 -0600 Subject: [PATCH] update --- defaults.inc | 2 +- inc/certs.inc | 28 ++++++++++++++++++---------- inc/services.inc | 8 ++++---- inc/status.inc | 28 ++++++++++++++-------------- nodemgmt-scripts.sh | 6 +++--- 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/defaults.inc b/defaults.inc index 635b0456..2b30972b 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ #!/usr/bin/env bash -VERS='4.15.15-11232023' +VERS='4.15.20-11232023' noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log ' CERT_DAEMON='/snap/bin/certbot' diff --git a/inc/certs.inc b/inc/certs.inc index 670cf6e3..dcb71e28 100755 --- a/inc/certs.inc +++ b/inc/certs.inc @@ -679,29 +679,37 @@ export PDNS_Ttl=60 scp -q /tmp/vcenter-update-ssl.sh root@${NM_VC_HOSTNAME}:/tmp/vcenter-update-ssl.sh ssh -q root@${NM_VC_HOSTNAME} "bash /tmp/vcenter-update-ssl.sh" - LIVEMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum /etc/vmware-rhttpproxy/ssl/rui.crt | cut -d\ -f1") - CURRENTMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum ${VC_CERT} | cut -d\ -f1") - if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then - echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}" - echo + if [ ! -f ${NM_VC_ACMEFOLDER}/${NM_VC_HOSTNAME}/${NM_VC_HOSTNAME}.cer ]; then + echo -e "${idsCL[LightYellow]}No certificate was created${idsCL[Default]}\n" exit 0 + elif [ $(expr `date +%s` - $(stat -c %Y ${NM_VC_ACMEFOLDER}/${NM_VC_HOSTNAME}/${NM_VC_HOSTNAME}.cer)) -ge 600 ]; then + echo -e "${idsCL[LightYellow]}The certificate was unable to be updated${idsCL[Default]}\n" + exit 0 + else + LIVEMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum /etc/vmware-rhttpproxy/ssl/rui.crt | cut -d\ -f1") + CURRENTMD5=$(ssh -q root@${NM_VC_HOSTNAME} "md5sum ${VC_CERT} | cut -d\ -f1") + if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then + echo -e "${idsCL[LightYellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}\n" + exit 0 + fi fi - + echo -e "${idsCL[LightGreen]}Updating certificates on vCenter... ${idsCL[Default]}" echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}" echo - ssh -q root@${NM_VC_HOSTNAME} "(printf '1\n%s\n' '${NM_VC_USER}'; sleep 1; printf '%s\n' '${NM_VC_PASS}'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '${VC_CERT}' '${VC_KEY}' '${VC_CHAIN}') | setsid /usr/lib/vmware-vmca/bin/certificate-manager" + # ssh -q root@${NM_VC_HOSTNAME} "(printf '1\n%s\n' '${NM_VC_USER}'; sleep 1; printf '%s\n' '${NM_VC_PASS}'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '${VC_CERT}' '${VC_KEY}' '${VC_CHAIN}') | setsid /usr/lib/vmware-vmca/bin/certificate-manager" - SENDNOTICE "vCenter SSL Updated" "Refresh/rescan any systems connecting to vcenter like Veeam" + SENDNOTICE "vCenter SSL Updated" "Refresh/Reconnect any systems connecting to vCenter, like Veeam Dummy!" + echo "$(date +%Y-%m-%d-%H-%M-%S) VCENTER SSL HAS BEEN UPDATED" >> ${NM_LOGFOLDER}/vc-ssl.log ssh -q root@${NM_VC_HOSTNAME} "rm -f /tmp/vcenter-update-ssl.sh" rm -f /tmp/vcenter-update-ssl.sh echo -e "\n${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}" - echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}\n" + echo -e "${idsCL[LightCyan]}Don't forget to redo the vCenter connection in Veeam${idsCL[Default]}\n" else - echo -e "${idsCL[Yellow]}vCenter info not configured in 'defaults.local.inc'${idsCL[Default]}\n" + echo -e "${idsCL[LightYellow]}vCenter info not configured in 'defaults.local.inc'${idsCL[Default]}\n" fi } diff --git a/inc/services.inc b/inc/services.inc index a01d739e..2ab3335e 100755 --- a/inc/services.inc +++ b/inc/services.inc @@ -22,7 +22,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ if [ "${gofor}" == "true" ]; then for stc in "${!servicestocheck}"; do if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then - echo "$(date) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${stc} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - Starting ${stc} on ${RUN_NODE_IP}, stopping on the other Nodes ..." >> ${NM_LOGFILE} [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.service pdnsadmin.socket" for nip in "${!hosts}"; do @@ -35,7 +35,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" [ "${stc}" = "headscale" ] && sleep 90s /bin/systemctl start ${stc} & - echo "$(date) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${NM_HOSTNAMES[${RUN_NODE_IP}]} - ... Done" >> ${NM_LOGFILE} fi done for docker in "${!dockerstocheck}"; do @@ -46,7 +46,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1 elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} fi @@ -57,7 +57,7 @@ SINGLE_SERVER_SERVICE_CHECK(){ [ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=auth_redis)" ] && ssh root@${nip} /usr/bin/docker stop auth_redis >/dev/null 2>&1 elif [ "${NM_DOCKER_COMPOSE_LOC[${docker}]}" != "" ]; then ssh root@${nip} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} fi fi diff --git a/inc/status.inc b/inc/status.inc index a064975f..fdfba876 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -92,7 +92,7 @@ STATUS(){ fi SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1 touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down) if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then @@ -118,7 +118,7 @@ STATUS(){ fi rm -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.* SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is back up" >> ${NM_LOGFILE} fi fi @@ -236,7 +236,7 @@ STATUS(){ [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" touch ${NM_TMPFOLDER}/${nip}~${srvc}.down # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} [ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} & elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then @@ -259,7 +259,7 @@ STATUS(){ if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime) || toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.* @@ -307,7 +307,7 @@ STATUS(){ # fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1 # touch ${NM_TMPFOLDER}/${nip}~${srvc}.down - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} + # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE} # # else # errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) @@ -336,7 +336,7 @@ STATUS(){ # toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down) # fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" - # echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + # echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} # # rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.down # rm -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime @@ -383,10 +383,10 @@ STATUS(){ if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then ${NCMD} docker start ${docker} >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${NM_LOGFILE} else ${NCMD} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker compose.yml up -d >/dev/null 2>&1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${NM_LOGFILE} fi sleep 10s if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then @@ -395,7 +395,7 @@ STATUS(){ fi SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed" rm -f ${NM_TMPFOLDER}/${nip}~${docker}.* - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE} else if [ "${ST_ACTION}" != "check" ]; then @@ -404,7 +404,7 @@ STATUS(){ if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1 touch ${NM_TMPFOLDER}/${nip}~${docker}.down - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${NM_LOGFILE} else errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.down) if [ $errtime -gt ${NM_RENOTIFY_TIMEOUT} ]; then @@ -443,7 +443,7 @@ STATUS(){ echo -e "${idsCL[Red]} - Node is down!${idsCL[Default]}" fi # SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_HOSTNAMES[${nip}]}[${nip}] is down" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - Node is down" >> ${NM_LOGFILE} else toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) @@ -544,7 +544,7 @@ STATUS(){ mv ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${NM_LOGFILE} else snderrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down)` # echo "HERE: $snderrtime" @@ -552,7 +552,7 @@ STATUS(){ toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)` SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) It has been down for $(SHOW_TIME ${toterrtime})" 1 - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, has now been down for: $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down fi fi @@ -564,7 +564,7 @@ It has been down for $(SHOW_TIME ${toterrtime})" 1 toterrtime=`expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.errtime)` SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]}) It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${NM_LOGFILE} fi rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl* fi diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 80e352c8..08d2c61d 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -843,11 +843,11 @@ GUI(){ touch "${NM_TMPFOLDER}/status-check.running" if [ "${2}" == "report" ]; then # start_status=`date +%s` - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} - echo -e "$(date) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE} + echo -e "$(date +%Y-%m-%d-%H-%M-%S) - Starting status check #############################\n" > ${NM_TMPFOLDER}/status-check.running STATUS report ${3} >> ${NM_TMPFOLDER}/status-check.running echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_TMPFOLDER}/status-check.running - echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} + echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE} rm -f ${NM_LOGFOLDER}/status-check.lastrun mv ${NM_TMPFOLDER}/status-check.running ${NM_LOGFOLDER}/status-check.lastrun