This commit is contained in:
2023-11-06 09:58:11 -06:00
parent 5e6764b45d
commit b2ba17490e
2 changed files with 12 additions and 11 deletions

View File

@@ -1,5 +1,5 @@
VERS='2.3.200-11052023' VERS='2.3.205-11062023'
noheader=' update service dailytemp ' noheader=' update service dailytemp '

View File

@@ -14,6 +14,7 @@ SHUTDOWNSERVERS_AT_LOAD=80 #percent
WAIT_TO_POWER_SERVERS=5 #minutes WAIT_TO_POWER_SERVERS=5 #minutes
LOG_INTERVAL_NORMAL=60 #minutes LOG_INTERVAL_NORMAL=60 #minutes
LOG_INTERVAL_ABNORMAL=30 #seconds LOG_INTERVAL_ABNORMAL=30 #seconds
LOG_INTERVAL_CHARGING=120 #seconds
# TEST=true #full # TEST=true #full
if [ "$TEST" != "" ]; then if [ "$TEST" != "" ]; then
@@ -42,14 +43,14 @@ PWO_START(){
BATTERY CHARGE: ${OFFSITE_BATT_CAP}%25 BATTERY CHARGE: ${OFFSITE_BATT_CAP}%25
BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins
AC VOLTAGE: ${OFFSITE_VOLTIN}'Volts AC VOLTAGE: ${OFFSITE_VOLTIN}'volts
$(date)" $(date)"
if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi
curl -sd "token=${PUSHOVER_APP_TOKEN}&user=${PUSHOVER_USER_TOKEN}&message=${MESSAGE}&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1 curl -sd "token=${PUSHOVER_APP_TOKEN}&user=${PUSHOVER_USER_TOKEN}&message=${MESSAGE}&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1
} }
if [ "${startup}" == "true" ]; then if [ "${startup}" == "true" ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Service startup" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Service startup" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE POWER MONITOR" "Service starting" SENDNOTICE "OFFSITE POWER MONITOR" "Service starting"
startup=false startup=false
fi fi
@@ -57,7 +58,7 @@ $(date)"
if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then
if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Offsite Power Restored" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Offsite Power Restored" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${WAIT_TO_POWER_SERVERS}'Mins before turning host(s) back on" SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${WAIT_TO_POWER_SERVERS}'Mins before turning host(s) back on"
fi fi
@@ -68,34 +69,34 @@ $(date)"
touch ${PW_TMPFOLDER}/power.offsite.restored touch ${PW_TMPFOLDER}/power.offsite.restored
elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${WAIT_TO_POWER_SERVERS}*60" | bc) ]; then elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage" SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage"
([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ipmitool -I lanplus -H 10.2.1.21 -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & ([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ipmitool -I lanplus -H 10.2.1.21 -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
rm -f ${PW_TMPFOLDER}/power.offsite.* rm -f ${PW_TMPFOLDER}/power.offsite.*
fi fi
fi fi
logtime=$(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) logtime=$(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER}))
if [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_ABNORMAL}" | bc) ]); then if [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${LOG_INTERVAL_CHARGING} ]); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Normal" >> ${PW_LOGPOWER} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Normal" >> ${PW_LOGPOWER}
fi fi
else else
if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low)) -gt 240 ]; then if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low)) -gt 240 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1 SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1
touch ${PW_TMPFOLDER}/power.offsite.ac.low touch ${PW_TMPFOLDER}/power.offsite.ac.low
fi fi
if [ ${OFFSITE_BATT_CAP%.*} -lt ${SHUTDOWNSERVERS_AT_LOAD} ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then if [ ${OFFSITE_BATT_CAP%.*} -lt ${SHUTDOWNSERVERS_AT_LOAD} ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering Down Hosts!" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering Down Hosts!" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1 SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
touch ${PW_TMPFOLDER}/power.offsite.off touch ${PW_TMPFOLDER}/power.offsite.off
([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ssh root@10.2.1.11 /vmfs/volumes/652c94ef-ee49895b-365d-00109b3fea40/.SCRIPTS/esxi-shutdown2.sh & ([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ssh root@10.2.1.11 /vmfs/volumes/652c94ef-ee49895b-365d-00109b3fea40/.SCRIPTS/esxi-shutdown2.sh &
fi fi
[ $(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) -gt $(echo "scale=2; ${LOG_INTERVAL_ABNORMAL}" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - OFF/LOW POWER!!" >> ${PW_LOGPOWER} [ $(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) -gt ${LOG_INTERVAL_ABNORMAL} ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - OFF/LOW POWER!!" >> ${PW_LOGPOWER}
fi fi
else else
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Restarting NUT service" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Restarting NUT service" >> ${PW_LOGFILE}
SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service" SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service"
/usr/local/etc/rc.d/nut.sh restart /usr/local/etc/rc.d/nut.sh restart
sleep 10s sleep 10s