voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update powerwall.sh

Browse Source
This commit is contained in:
David Schroeder
2023-11-04 17:24:17 -05:00
parent 9749095e58
commit 6d0ba64a67
1 changed files with 93 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
powerwall.sh
View File

@@ -84,7 +84,7 @@ CHECKTEMP(){
# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
# SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
# echo "(${datetime}) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
# touch ${PW_TMPFOLDER}/${EHOST}.down
# fi
fi
@@ -215,7 +215,7 @@ CHECKTEMP_SERVICE(){
# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
echo "(${datetime}) - ${SENSORa} - Sensor is back online" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
fi
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
@@ -271,12 +271,11 @@ CHECKTEMP_SERVICE(){
((t++))
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "(${datetime}) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
@@ -301,9 +300,9 @@ CHECKTEMP_SERVICE(){
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "(${datetime}) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
SENDNOTICE "${SENSOR} TEMP WARNING" "WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
last_temp[${SENSOR}]=${temp_f}
echo "(${datetime}) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
@@ -316,15 +315,15 @@ CHECKTEMP_SERVICE(){
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ ${temp_diff} -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
SENDNOTICE "${SENSOR} TEMP CRITICAL" "CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "(${datetime}) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo -e "(${datetime}) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
SHUTDOWN_MAIN ${SENSOR} &
echo -e "(${datetime}) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
@@ -336,32 +335,32 @@ Shutting down servers!!" 1
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ ${temp_diff} -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "(${datetime}) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "(${datetime}) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
fi
else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "(${datetime}) NORMAL TEMP: ${temp_f}'F
SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "(${datetime}) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
# SENDNOTICE "${SENSOR} TEMP NORMAL" "(${datetime}) Service Startup
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
# NORMAL TEMP: ${temp_f}'F"
echo -e "(${datetime}) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${relog} -eq 1 ]; then
echo "(${datetime}) - ${temp_f}'F - Normal Temp" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${PW_TMPFOLDER}/temp.*
@@ -391,23 +390,23 @@ Previous Temp: ${last_temp[${SENSOR}]}'F"
fi
entityid=${entityid[1]}
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','${datetime}','${temp_f}')"
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ]; then
QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','${datetime}','${FanSpeed}')"
QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
${mysql_conn} -e "${QRY2}"
fi
elif [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}','${temp_h}')"
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}')"
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${PW_TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} Sensor ERROR" "(${datetime}) ERROR reading sensor data" 1
echo "(${datetime}) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
@@ -422,7 +421,7 @@ Previous Temp: ${last_temp[${SENSOR}]}'F"
elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
echo "(${datetime}) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSORa}.down
fi
@@ -628,7 +627,6 @@ CHECKPOWER_SERVICE(){
voltstatus=0
battstatus=0
while true; do
datetime=`date +'%Y-%m-%d %H:%M:%S'`
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
@@ -637,7 +635,7 @@ CHECKPOWER_SERVICE(){
touch ${PW_TMPFOLDER}/power.ups.low
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "(${datetime}) UPS has no power for 1min, shutting all servers down!" 1
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
fi
@@ -651,56 +649,61 @@ CHECKPOWER_SERVICE(){
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
if [ ${voltstatus} -eq 3 ]; then
echo "(${datetime}) - ${INPUTACV}'V - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "(${datetime}) Normal voltage detected
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${INPUTACV}'V"
echo -e "(${datetime}) - ${INPUTACV}'V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
battstatus=0
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "(${datetime}) - ${INPUTACV}'V - Normal Voltage" >> ${PW_LOGPOWER}
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - Normal Voltage" >> ${PW_LOGPOWER}
if [ ${voltstatus} -eq 0 ]; then
SENDNOTICE "Power Nominal" "(${datetime}) Service Startup
SENDNOTICE "Power Nominal" "Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}'V
BATT VOLTAGE: ${BATTVOLT}'V"
echo "(${datetime}) - Input: ${INPUTACV}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
echo -e "(${datetime}) - ${INPUTACV}'V / ${BATTVOLT}'V - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V / ${BATTVOLT}'V - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
fi
fi
voltstatus=1
rm -f ${PW_TMPFOLDER}/power.ac.*
rm -f ${PW_TMPFOLDER}/power.*
else
echo "(${datetime}) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo -e "(${datetime}) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
if [ ${voltstatus} -lt 3 ]; then
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "(${datetime}) Power off or low voltage detected
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "Power off or low voltage detected
VOLTAGE: ${INPUTACV}'V
BATT VOLTAGE: ${BATTVOLT}'V" 1
last_battvolt=${BATTVOLT}
voltstatus=3
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 300 ]; then
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
SHUTDOWN_SERVER 10.10.2.10 &
touch ${PW_TMPFOLDER}/power.ac.conserve
fi
voltstatus=3
touch ${PW_TMPFOLDER}/power.ac.low
fi
if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then
echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then
echo "(${datetime}) - ${BATTVOLT}'V - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "(${datetime}) Battery Voltage LOW: ${BATTVOLT}'V
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'V
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
battstatus=2
fi
elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then
echo "(${datetime}) - ${BATTVOLT}'V - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "(${datetime}) Battery Voltage REALLY LOW: ${BATTVOLT}'V
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'V
Shutting down all servers" 1
SHUTDOWN_SYS POWER &
@@ -709,20 +712,19 @@ Shutting down all servers" 1
fi
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "(${datetime}) - ${BATTVOLT}'V - Battery Voltage" >> ${PW_LOGPOWER}
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Battery Voltage" >> ${PW_LOGPOWER}
if [ ${battstatus} -eq 0 ]; then
echo "(${datetime}) - Battery: ${BATTVOLT}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
fi
rm -f ${PW_TMPFOLDER}/power.dc.*
battstatus=1
fi
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} > 0") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "(${datetime}) Battery Voltage: ${BATTVOLT}'V"
echo -e "(${datetime}) - ${BATTVOLT}'V - Battery Voltage Change" >> ${PW_LOGPOWER}
if (( $(bc <<<"${volt_diff} > 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'V"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
else
@@ -738,20 +740,20 @@ Shutting down all servers" 1
if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then
if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "(${datetime}) Normal voltage detected
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${OFFSITE_VOLTIN}'V"
rm -f ${PW_TMPFOLDER}/power.offsite.*
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - Offsite Normal Voltage" >> ${PW_LOGPOWER}
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - Offsite Normal Voltage" >> ${PW_LOGPOWER}
fi
else
echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 300 ]; then
SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "(${datetime}) Power off or low voltage detected
SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
VOLTAGE: ${OFFSITE_VOLTIN}'V" 1
touch ${PW_TMPFOLDER}/power.offsite.ac.low
fi
@@ -770,7 +772,7 @@ VOLTAGE: ${OFFSITE_VOLTIN}'V" 1
POWERLOGGER_SERVICE(){
while true; do
datetime=`date +'%Y-%m-%d %H:%M:%S'`
LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
@@ -782,14 +784,14 @@ POWERLOGGER_SERVICE(){
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${INPUTV}','${INPUTA}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${LogDataTime}','${INPUTV}','${INPUTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('4','${datetime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('4','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${BATTV}','${BATTA}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${LogDataTime}','${BATTV}','${BATTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
@@ -800,7 +802,7 @@ POWERLOGGER_SERVICE(){
SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${datetime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
@@ -809,9 +811,9 @@ POWERLOGGER_SERVICE(){
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
if [ "${SVRRM_VOLT}" != "" ]; then
APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${datetime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
else
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${datetime}','${APCPDU_AMP}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
fi
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
@@ -820,7 +822,7 @@ POWERLOGGER_SERVICE(){
NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${datetime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
@@ -832,11 +834,11 @@ POWERLOGGER_SERVICE(){
OFFSITE_WATTS=`echo "scale=1; ${OFFSITE_LOAD}*900" | bc`
OFFSITE_AMPS=`echo "scale=1; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('15','${datetime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('15','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then
sleep 1m
else
sleep 5m
@@ -917,10 +919,9 @@ SHUTDOWN_MAIN(){
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "MAIN SERVERS SHUTDOWN" "(${datetime}) MAIN servers have been shutdown
${reason}" 1
echo -e "(${datetime}) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown
${reason}" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
fi
echo
@@ -941,11 +942,10 @@ SHUTDOWN_SYS(){
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "SYS SERVERS SHUTDOWN" "(${datetime}) SYS servers have been shutdown
SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown
${reason}" 1
echo -e "(${datetime}) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
echo -e "(${datetime}) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
# curl --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd
@@ -953,16 +953,16 @@ SHUTDOWN_SYS(){
}
SHUTDOWN_CRIT(){
if [ "${1}" == "Powerwall-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) Powerwall System shutting down" 1
echo -e "(${datetime}) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "(${datetime}) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
SENDNOTICE "${SENSOR} TEMP CRITICAL" "Powerwall System shutting down" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
sleep 1s
/sbin/poweroff
elif [ "${1}" == "OctoPI-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) OctoPI System shutting down" 1
echo -e "(${datetime}) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "(${datetime}) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
SENDNOTICE "${SENSOR} TEMP CRITICAL" "OctoPI System shutting down" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
sleep 1s
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
@@ -971,7 +971,7 @@ SHUTDOWN_CRIT(){
SHUTDOWN_OFFSITE(){
if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "(${datetime}) Shutting down '${PW_ESXI_HOST_NAMES[${offsite_host}]}'" 1
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down '${PW_ESXI_HOST_NAMES[${offsite_host}]}'" 1
SHUTDOWN_SERVER ${offsite_host} &
done
touch ${PW_FOLDER}/shutdown.offsite
@@ -980,7 +980,7 @@ SHUTDOWN_OFFSITE(){
SHUTDOWN_SERVERS(){
# TEST=true
echo "(${datetime}) - Shutting down ${1} servers" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE}
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
if [ "${1^^}" == "MAIN" ]; then
@@ -1195,7 +1195,7 @@ SHUTDOWN_SERVER(){
DIVIDER . yellow 75
echo
echo "(${datetime}) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE}
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
@@ -1410,7 +1410,7 @@ TEST(){
echo "Average: ${average}'F"
echo
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('$(date +'%Y-%m-%d %H:%M:%S')','${INVSTATSINFO}')"
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
end=`date +%s`