From 6d0ba64a67d4c22e090887b45997264545304ec5 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Sat, 4 Nov 2023 17:24:17 -0500 Subject: [PATCH] Update powerwall.sh --- powerwall.sh | 186 +++++++++++++++++++++++++-------------------------- 1 file changed, 93 insertions(+), 93 deletions(-) diff --git a/powerwall.sh b/powerwall.sh index d307eeba..29de10ae 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -84,7 +84,7 @@ CHECKTEMP(){ # if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then # SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1 - # echo "(${datetime}) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp} + # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp} # touch ${PW_TMPFOLDER}/${EHOST}.down # fi fi @@ -215,7 +215,7 @@ CHECKTEMP_SERVICE(){ # if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online" - echo "(${datetime}) - ${SENSORa} - Sensor is back online" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSORa}.down fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then @@ -271,12 +271,11 @@ CHECKTEMP_SERVICE(){ ((t++)) fi - datetime=`date +'%Y-%m-%d %H:%M:%S'` logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR} if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again" - echo "(${datetime}) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading fi @@ -301,9 +300,9 @@ CHECKTEMP_SERVICE(){ if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.warn if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP WARNING" "(${datetime}) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 + SENDNOTICE "${SENSOR} TEMP WARNING" "WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 last_temp[${SENSOR}]=${temp_f} - echo "(${datetime}) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp} # iDS-vMS-Offsite Fan Issue Workaround if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then @@ -316,15 +315,15 @@ CHECKTEMP_SERVICE(){ elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}_sys") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.crit if [ ${temp_diff} -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') + SENDNOTICE "${SENSOR} TEMP CRITICAL" "CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') Shutting down servers!!" 1 - echo "(${datetime}) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} - echo -e "(${datetime}) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=${temp_f} if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then SHUTDOWN_MAIN ${SENSOR} & - echo -e "(${datetime}) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 @@ -336,32 +335,32 @@ Shutting down servers!!" 1 elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}_sys") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.critsys if [ ${temp_diff} -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 - echo "(${datetime}) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} - echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} + SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=${temp_f} SHUTDOWN_SYS ${SENSOR} TEMP & SHUTDOWN_CRIT ${SENSOR} & - echo -e "(${datetime}) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} fi else if [ ${last_temp[${SENSOR}]} -gt 1 ]; then - SENDNOTICE "${SENSOR} BACK TO NORMAL" "(${datetime}) NORMAL TEMP: ${temp_f}'F + SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}'F Previous Temp: ${last_temp[${SENSOR}]}'F" - echo -e "(${datetime}) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} - echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE} [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then - # SENDNOTICE "${SENSOR} TEMP NORMAL" "(${datetime}) Service Startup + # SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup # NORMAL TEMP: ${temp_f}'F" - echo -e "(${datetime}) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} - echo -e "(${datetime}) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* elif [ ${relog} -eq 1 ]; then - echo "(${datetime}) - ${temp_f}'F - Normal Temp" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Normal Temp" >> ${logtemp} fi rm -f ${PW_TMPFOLDER}/temp.* @@ -391,23 +390,23 @@ Previous Temp: ${last_temp[${SENSOR}]}'F" fi entityid=${entityid[1]} - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','${datetime}','${temp_f}')" + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ]; then - QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','${datetime}','${FanSpeed}')" + QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')" ${mysql_conn} -e "${QRY2}" fi elif [ "${temp_h}" != "" ]; then - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}','${temp_h}')" + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')" else - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}')" + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" fi ${mysql_conn} -e "${QRY}" touch ${PW_TMPFOLDER}/${SENSOR}.insert fi else if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then - SENDNOTICE "${SENSOR} Sensor ERROR" "(${datetime}) ERROR reading sensor data" 1 - echo "(${datetime}) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} + SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1 + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} touch ${PW_TMPFOLDER}/${SENSOR}-error.reading fi @@ -422,7 +421,7 @@ Previous Temp: ${last_temp[${SENSOR}]}'F" elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1 - echo "(${datetime}) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp} touch ${PW_TMPFOLDER}/${SENSORa}.down fi @@ -628,7 +627,6 @@ CHECKPOWER_SERVICE(){ voltstatus=0 battstatus=0 while true; do - datetime=`date +'%Y-%m-%d %H:%M:%S'` INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json") SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //') @@ -637,7 +635,7 @@ CHECKPOWER_SERVICE(){ touch ${PW_TMPFOLDER}/power.ups.low elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then - SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "(${datetime}) UPS has no power for 1min, shutting all servers down!" 1 + SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1 fi @@ -651,56 +649,61 @@ CHECKPOWER_SERVICE(){ if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then if [ ${voltstatus} -eq 3 ]; then - echo "(${datetime}) - ${INPUTACV}'V - Back to Normal Voltage" >> ${PW_LOGPOWER} - SENDNOTICE "POWER NOMINAL AGAIN" "(${datetime}) Normal voltage detected + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - Back to Normal Voltage" >> ${PW_LOGPOWER} + SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected VOLTAGE: ${INPUTACV}'V" - echo -e "(${datetime}) - ${INPUTACV}'V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE} battstatus=0 else - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "(${datetime}) - ${INPUTACV}'V - Normal Voltage" >> ${PW_LOGPOWER} + [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - Normal Voltage" >> ${PW_LOGPOWER} if [ ${voltstatus} -eq 0 ]; then - SENDNOTICE "Power Nominal" "(${datetime}) Service Startup + SENDNOTICE "Power Nominal" "Service Startup Normal voltage detected VOLTAGE: ${INPUTACV}'V BATT VOLTAGE: ${BATTVOLT}'V" - echo "(${datetime}) - Input: ${INPUTACV}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} - echo -e "(${datetime}) - ${INPUTACV}'V / ${BATTVOLT}'V - Service Startup - Normal Voltage" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V / ${BATTVOLT}'V - Service Startup - Normal Voltage" >> ${PW_LOGFILE} fi fi voltstatus=1 - rm -f ${PW_TMPFOLDER}/power.ac.* + rm -f ${PW_TMPFOLDER}/power.* else - echo "(${datetime}) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER} - echo -e "(${datetime}) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'V - POWER OFF/LOW POWER" >> ${PW_LOGFILE} if [ ${voltstatus} -lt 3 ]; then - SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "(${datetime}) Power off or low voltage detected + SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "Power off or low voltage detected VOLTAGE: ${INPUTACV}'V BATT VOLTAGE: ${BATTVOLT}'V" 1 last_battvolt=${BATTVOLT} + voltstatus=3 + touch ${PW_TMPFOLDER}/power.ac.low + fi + if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 300 ]; then + SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1 + SHUTDOWN_SERVER 10.10.2.10 & + touch ${PW_TMPFOLDER}/power.ac.conserve fi - voltstatus=3 - touch ${PW_TMPFOLDER}/power.ac.low fi if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then - echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER} - echo -e "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} touch ${PW_TMPFOLDER}/power.dc.low if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then - echo "(${datetime}) - ${BATTVOLT}'V - Shutting down main servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "(${datetime}) Battery Voltage LOW: ${BATTVOLT}'V + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Shutting down main servers..." >> ${PW_LOGPOWER} + SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'V Shutting down main servers" 1 SHUTDOWN_MAIN POWER & battstatus=2 fi elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then - echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER} - echo "(${datetime}) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY Voltage" >> ${PW_LOGPOWER} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE} touch ${PW_TMPFOLDER}/power.dc.low if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then - echo "(${datetime}) - ${BATTVOLT}'V - Shutting down all remaining servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "(${datetime}) Battery Voltage REALLY LOW: ${BATTVOLT}'V + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Shutting down all remaining servers..." >> ${PW_LOGPOWER} + SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'V Shutting down all servers" 1 SHUTDOWN_SYS POWER & @@ -709,20 +712,19 @@ Shutting down all servers" 1 fi else - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "(${datetime}) - ${BATTVOLT}'V - Battery Voltage" >> ${PW_LOGPOWER} + [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Battery Voltage" >> ${PW_LOGPOWER} if [ ${battstatus} -eq 0 ]; then - echo "(${datetime}) - Battery: ${BATTVOLT}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} fi - rm -f ${PW_TMPFOLDER}/power.dc.* battstatus=1 fi volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc` (( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc` - if (( $(bc <<<"${volt_diff} > 0") )); then - SENDNOTICE "BATTERY VOLTAGE CHANGE" "(${datetime}) Battery Voltage: ${BATTVOLT}'V" - echo -e "(${datetime}) - ${BATTVOLT}'V - Battery Voltage Change" >> ${PW_LOGPOWER} + if (( $(bc <<<"${volt_diff} > 0.5") )); then + SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'V" + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'V - Battery Voltage Change" >> ${PW_LOGPOWER} last_battvolt=${BATTVOLT} fi else @@ -738,20 +740,20 @@ Shutting down all servers" 1 if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then - echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER} - echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "(${datetime}) Normal voltage detected + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected VOLTAGE: ${OFFSITE_VOLTIN}'V" rm -f ${PW_TMPFOLDER}/power.offsite.* else - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - Offsite Normal Voltage" >> ${PW_LOGPOWER} + [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - Offsite Normal Voltage" >> ${PW_LOGPOWER} fi else - echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER} - echo "(${datetime}) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'V - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE} if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 300 ]; then - SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "(${datetime}) Power off or low voltage detected + SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected VOLTAGE: ${OFFSITE_VOLTIN}'V" 1 touch ${PW_TMPFOLDER}/power.offsite.ac.low fi @@ -770,7 +772,7 @@ VOLTAGE: ${OFFSITE_VOLTIN}'V" 1 POWERLOGGER_SERVICE(){ while true; do - datetime=`date +'%Y-%m-%d %H:%M:%S'` + LogDataTime=`date +'%Y-%m-%d %H:%M:%S'` INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json") if [ "${INVSTATSINFO}" != "" ]; then @@ -782,14 +784,14 @@ POWERLOGGER_SERVICE(){ BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'` BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${INPUTV}','${INPUTA}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${LogDataTime}','${INPUTV}','${INPUTA}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('4','${datetime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('4','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${BATTV}','${BATTA}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${LogDataTime}','${BATTV}','${BATTA}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" - QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')" + QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" fi @@ -800,7 +802,7 @@ POWERLOGGER_SERVICE(){ SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //') SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${datetime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" fi @@ -809,9 +811,9 @@ POWERLOGGER_SERVICE(){ APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc` if [ "${SVRRM_VOLT}" != "" ]; then APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${datetime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')" else - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${datetime}','${APCPDU_AMP}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')" fi ${mysql_conn} -e "USE servermonitor; ${QRY}" fi @@ -820,7 +822,7 @@ POWERLOGGER_SERVICE(){ NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc` NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${datetime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" fi @@ -832,11 +834,11 @@ POWERLOGGER_SERVICE(){ OFFSITE_WATTS=`echo "scale=1; ${OFFSITE_LOAD}*900" | bc` OFFSITE_AMPS=`echo "scale=1; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('15','${datetime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')" + QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('15','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" fi - if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then + if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then sleep 1m else sleep 5m @@ -917,10 +919,9 @@ SHUTDOWN_MAIN(){ fi if [ "${sendnotice}" == "true"]; then - datetime=`date +'%Y-%m-%d %H:%M:%S'` - SENDNOTICE "MAIN SERVERS SHUTDOWN" "(${datetime}) MAIN servers have been shutdown - ${reason}" 1 - echo -e "(${datetime}) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE} + SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown +${reason}" 1 + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE} fi echo @@ -941,11 +942,10 @@ SHUTDOWN_SYS(){ fi if [ "${sendnotice}" == "true"]; then - datetime=`date +'%Y-%m-%d %H:%M:%S'` - SENDNOTICE "SYS SERVERS SHUTDOWN" "(${datetime}) SYS servers have been shutdown + SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown ${reason}" 1 - echo -e "(${datetime}) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE} - echo -e "(${datetime}) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} fi # curl --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd @@ -953,16 +953,16 @@ SHUTDOWN_SYS(){ } SHUTDOWN_CRIT(){ if [ "${1}" == "Powerwall-CPU" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) Powerwall System shutting down" 1 - echo -e "(${datetime}) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} - echo -e "(${datetime}) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE} + SENDNOTICE "${SENSOR} TEMP CRITICAL" "Powerwall System shutting down" 1 + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE} sleep 1s /sbin/poweroff elif [ "${1}" == "OctoPI-CPU" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "(${datetime}) OctoPI System shutting down" 1 - echo -e "(${datetime}) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} - echo -e "(${datetime}) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE} + SENDNOTICE "${SENSOR} TEMP CRITICAL" "OctoPI System shutting down" 1 + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} + echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE} sleep 1s ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff @@ -971,7 +971,7 @@ SHUTDOWN_CRIT(){ SHUTDOWN_OFFSITE(){ if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do - SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "(${datetime}) Shutting down '${PW_ESXI_HOST_NAMES[${offsite_host}]}'" 1 + SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down '${PW_ESXI_HOST_NAMES[${offsite_host}]}'" 1 SHUTDOWN_SERVER ${offsite_host} & done touch ${PW_FOLDER}/shutdown.offsite @@ -980,7 +980,7 @@ SHUTDOWN_OFFSITE(){ SHUTDOWN_SERVERS(){ # TEST=true - echo "(${datetime}) - Shutting down ${1} servers" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE} [ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE} if [ "${1^^}" == "MAIN" ]; then @@ -1195,7 +1195,7 @@ SHUTDOWN_SERVER(){ DIVIDER . yellow 75 echo - echo "(${datetime}) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE} if [ "${ESXIHOST}" == "10.10.2.10" ]; then echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... " @@ -1410,7 +1410,7 @@ TEST(){ echo "Average: ${average}'F" echo - # QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')" + # QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('$(date +'%Y-%m-%d %H:%M:%S')','${INVSTATSINFO}')" # ${mysql_conn} -e "USE servermonitor; ${QRY}" end=`date +%s`