voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-11 15:34:54 -06:00
parent 0d5c575de2
commit 552bd90054
2 changed files with 102 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
offsite-power-check.sh
View File

@@ -97,6 +97,9 @@ $(date)"
for host in ${PW_OFFSITEHOSTS_IDRAC}; do
if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then
ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
else
echo "TESTMODE: Powering up '${host}'"
fi

204
powerwall.sh
View File

@@ -240,22 +240,6 @@ CHECKTEMP_SERVICE(){
[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
t=1
# # iDS-vMS-Offsite Fan Issue Workaround
# if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
# # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
# # FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
# if [ "${vHOSTFAN}" != "" ]; then
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
# if [ ${FanSpeed} -gt 92 ]; then
# SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30%25" 1
# ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
# ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
# FanSpeed=30
# fi
# fi
# fi
else
SENSORS_CHECK=(${SENSORa})
fi
@@ -327,15 +311,10 @@ CHECKTEMP_SERVICE(){
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
fi
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc`
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
[ ! "${last_temp[${SENSOR}]}" ] && last_temp[${SENSOR}]=${temp_f}
temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc`
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
@@ -360,27 +339,30 @@ CHECKTEMP_SERVICE(){
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
touch ${PW_TMPFOLDER}/temp.critsys
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
# SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
# SHUTDOWN_SERVER ${SENSORa} &
# touch ${PW_TMPFOLDER}/power.${SENSORa}.off
elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
SHUTDOWN_SERVER ${SENSORa} &
touch ${PW_TMPFOLDER}/power.${SENSORa}.off
fi
fi
fi
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
touch ${PW_TMPFOLDER}/temp.critsys
[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
fi
fi
else
if [ $(bc -l <<< "${last_temp[${SENSOR}]} > 1") -eq 1 ]; then
@@ -388,14 +370,14 @@ CHECKTEMP_SERVICE(){
Previous Temp: ${last_temp[${SENSOR}]}${GAUGESH}"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR}]}${GAUGESH}" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
# [ "$(compgen -G "${PW_TMPFOLDER}/shutdown.*")" != "" ] && rm -f ${PW_TMPFOLDER}/shutdown.*
elif [ "${last_temp[${SENSOR}]}" == "1" ]; then
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
# NORMAL TEMP: ${temp_f}${GAUGESH}"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
# [ "$(compgen -G "${PW_TMPFOLDER}/shutdown.*")" != "" ] && rm -f ${PW_TMPFOLDER}/shutdown.*
elif [ "${relog}" == "1" ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
@@ -844,8 +826,8 @@ CHECKPOWER(){
}
CHECKPOWER_SERVICE(){
voltstatus=0
battstatus=0
touch ${PW_TMPFOLDER}/power.ac.start
touch ${PW_TMPFOLDER}/power.dc.start
while true; do
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
@@ -857,6 +839,10 @@ CHECKPOWER_SERVICE(){
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
######################### FINISH
fi
elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
@@ -866,16 +852,17 @@ CHECKPOWER_SERVICE(){
if [ "${INVSTATSINFO}" != "" ]; then
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
[ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT}
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
if [ ${voltstatus} -eq 3 ]; then
[ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT}
if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${INPUTACV}'volts"
battstatus=0
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
if [ ${voltstatus} -eq 0 ]; then
if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then
SENDNOTICE "Power Nominal" "Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}'volts
@@ -884,27 +871,27 @@ BATT VOLTAGE: ${BATTVOLT}'volts"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
fi
fi
voltstatus=1
rm -f ${PW_TMPFOLDER}/power.ac.*
if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then
rm -f ${PW_TMPFOLDER}/power.ac.*
touch ${PW_TMPFOLDER}/power.ac.good
fi
# UPDATE APC-PDU LINE TO LINE VOLTAGE
CURRENT_VOLTAGE=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //')
SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'`
[ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER}
[ "${SET_VOLTAGE}" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER}
else
if [ ${voltstatus} -lt 3 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGFILE}
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.ac.good ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts" 1
last_battvolt=${BATTVOLT}
voltstatus=3
rm -f ${PW_TMPFOLDER}/power.ac.good
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 180 ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGFILE}
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 120 ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
SHUTDOWN_SERVER ${PW_HOST_IDRACS['iDS-vMS-Host0']} &
@@ -912,50 +899,56 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1
fi
fi
if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
battstatus=2
fi
elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all servers" 1
SHUTDOWN_SYS POWER &
battstatus=3
fi
else
if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
if [ ${battstatus} -eq 0 ]; then
if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.dc.start
fi
battstatus=1
if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then
rm -f ${PW_TMPFOLDER}/power.dc.*
touch ${PW_TMPFOLDER}/power.dc.good
fi
else
[ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow
if [ ! -f ${PW_TMPFOLDER}/power.sys.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all remaining servers" 1
SHUTDOWN_SYS POWER &
fi
elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low
if [ ! -f ${PW_TMPFOLDER}/power.main.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
fi
fi
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} > 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
fi
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} > 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
else
sleep 20s
fi
@@ -1212,7 +1205,7 @@ SHUTDOWN_SYS(){
if [ "${sendnotice}" == "true"]; then
SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown
${reason}" 1
${reason}" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
@@ -1243,12 +1236,12 @@ SHUTDOWN_CRIT(){
fi
}
SHUTDOWN_OFFSITE(){
if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
if [ ! -f ${PW_TMPFOLDER}/shutdown.offsite ]; then
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1
for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
SHUTDOWN_SERVER ${offsite_host} &
done
touch ${PW_FOLDER}/shutdown.offsite
touch ${PW_TMPFOLDER}/shutdown.offsite
fi
}
@@ -1257,8 +1250,8 @@ SHUTDOWN_SERVERS(){
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
if [ "${1^^}" == "MAIN" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.main ]; then
touch ${PW_FOLDER}/shutdown.main
if [ ! -f ${PW_TMPFOLDER}/power.main.off ]; then
touch ${PW_TMPFOLDER}/power.main.off
if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
@@ -1365,9 +1358,9 @@ SHUTDOWN_SERVERS(){
fi
elif [ "${1^^}" == "SYS" ]; then
[ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP
if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then
touch ${PW_FOLDER}/shutdown.sys
[ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP
if [ ! -f ${PW_TMPFOLDER}/power.sys.off ]; then
touch ${PW_TMPFOLDER}/power.sys.off
if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
@@ -1450,10 +1443,11 @@ SHUTDOWN_SERVERS(){
fi
elif [ "${1^^}" == "ALL" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.all ]; then
touch ${PW_FOLDER}/shutdown.all
if [ ! -f ${PW_TMPFOLDER}/power.all.off ]; then
touch ${PW_TMPFOLDER}/power.all.off
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
@@ -1559,8 +1553,8 @@ SHUTDOWN_SERVERS(){
fi
elif [ "${1^^}" == "MAINSITE" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.mainsite ]; then
touch ${PW_FOLDER}/shutdown.mainsite
if [ ! -f ${PW_TMPFOLDER}/power.main.offsite ]; then
touch ${PW_TMPFOLDER}/power.main.offsite
if [ "${2}" == "TEMP" ] && [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
@@ -1667,8 +1661,8 @@ SHUTDOWN_SERVERS(){
fi
elif [ "${1^^}" == "OFFSITE" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
touch ${PW_FOLDER}/shutdown.offsite
if [ ! -f ${PW_TMPFOLDER}/shutdown.offsite ]; then
touch ${PW_TMPFOLDER}/shutdown.offsite
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"