diff --git a/defaults.inc b/defaults.inc index c197df6e..4cbbdf3d 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='2.0-05142023' +VERS='2.05-05142023' noheader=' update ' @@ -17,6 +17,8 @@ FOLDER='/opt/idssys/powerwall' logfile=${FOLDER}/logfile logacv=${FOLDER}/log-acv [ ! -f ${logacv} ] && touch ${logacv} +logtemp=${FOLDER}/log-temp +[ ! -f ${logtemp} ] && touch ${logtemp} declare -A TEMP_THRESHOLDS TEMP_THRESHOLDS['ServerRoomTH']="83,88,95" diff --git a/powerwall.sh b/powerwall.sh index 61914d19..3c5319eb 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # powerwall - CLI commands to control VM guest power -action="$1" +action="${1}" source /opt/idssys/powerwall/defaults.inc source /opt/idssys/powerwall/powerwall.conf @@ -74,84 +74,93 @@ CHECKTEMPSENSOR(){ } CHECKTEMP_SERVICE(){ - logsrvc=${FOLDER}/log-${1} - [ ! -f ${logsrvc} ] && touch ${logsrvc} - last_temp=0 - mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` + cw=20 + declare -A last_temp while true; do - ${mqtt_conn} -t tele/${1}/SENSOR | while read -r mqtt_message; do - echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h - echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c - echo $mqtt_message | cut -d'T' -f 2 | cut -d"\"" -f 3 | read temp_d - echo $mqtt_message | cut -d'T' -f 3 | cut -d"\"" -f 1 | read temp_t - # datetime="${temp_d} ${temp_t}" - # datetime=$(TZ=UTC0 date -d "${datetime}" +%s) - # datetime=`date -d @${datetime} +'%Y-%m-%d %H:%M:%S'` + for SENSOR in ${!SENSOR_ID[@]}; do + if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then + mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` + echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h + echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c + + elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then + temp_c=$(vcgencmd measure_temp) + temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} + temp_h='' + fi datetime=`date +'%Y-%m-%d %H:%M:%S'` if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` - - #echo "$temp_c -> $temp_f" - - echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn - echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit - echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys - - temp_diff=$((${last_temp%.*} - ${temp_f%.*})) - [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) - - if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then - if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then - SENDNOTICE "${1} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1 - echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc} - last_temp=$temp_f - fi - echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}°)" >> ${logsrvc} + + echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn + echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit + echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys + + if [ "${last_temp[${SENSOR}]}" != ""]; then + temp_diff=$((${last_temp[${SENSOR}]%.*} - ${temp_f%.*})) + [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) + else + last_temp[${SENSOR}]=0 + temp_diff=0 + fi + if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1 + echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} + last_temp[${SENSOR}]=$temp_f + fi + echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}°)" >> ${logtemp} + elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then - echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logsrvc} - if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then - SENDNOTICE "${1} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" 1 - echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc} - last_temp=$temp_f + echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logtemp} + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°) +Shutting down servers!!" 1 + echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} + last_temp[${SENSOR}]=$temp_f # SHUTDOWN_SERVERS MAIN SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1 - echo -e "($datetime) - ${temp_f}F - Main servers have been shutdown" >> ${logsrvc} + echo -e "($datetime) - ${SENSOR} - ${temp_f}F - Main servers have been shutdown" >> ${logtemp} fi elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then - echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logsrvc} - if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then - SENDNOTICE "${1} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" 1 - echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc} - last_temp=$temp_f + echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°) +Shutting down system servers!!" 1 + echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} + last_temp[${SENSOR}]=$temp_f # SHUTDOWN_SERVERS SYS SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1 - echo -e "($datetime) - ${temp_f}F - System servers have been shutdown" >> ${logsrvc} + echo -e "($datetime) - ${SENSOR} - ${temp_f}F - System servers have been shutdown" >> ${logtemp} fi else - if [ $last_temp -gt 1 ]; then - SENDNOTICE "${1} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F" - echo -e "($datetime) - ${temp_f}F - Back to NORMAL TEMP - Previous Temp: ${last_temp}F" >> ${logsrvc} - echo "($datetime) alert sent" >> ${logsrvc} + if [ ${last_temp[${SENSOR}]} -gt 1 ]; then + SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp[${SENSOR}]}°F" + echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} + echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} elif [ $last_temp -eq 0 ]; then - SENDNOTICE "${1} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F" - echo -e "($datetime) - ${temp_f}F - Service Startup - NORMAL TEMP" >> ${logsrvc} + SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F" + echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} fi - last_temp=1 - errtime=$(expr `date +%s` - $(stat -c %Y ${logsrvc})) - [ $errtime -ge 3600 ] && echo "($datetime) - ${temp_f}F - Normal Temp" >> ${logsrvc} + last_temp[${SENSOR}]=1 + errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp})) + [ $errtime -ge 3600 ] && echo "($datetime) - ${SENSOR} - ${temp_f}F - Normal Temp" >> ${logtemp} fi - - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')" + + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" ${mysql_conn} -e "${QRY}" else - SENDNOTICE "${1} ERROR" "($datetime) ERROR reading temperature" 1 - echo "($datetime) alert sent" >> ${logsrvc} + SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1 + echo "($datetime) - ${SENSOR} - alert sent" >> ${logtemp} fi - done - sleep 10 + + done + + sleep 10 + done # & } @@ -187,14 +196,17 @@ CHECKACV_SERVICE(){ if [ ${INPUTACV} -ge ${min_acvolt} ]; then if [ $voltstatus -eq 3 ]; then echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv} - SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V" + SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected +VOLTAGE: ${INPUTACV}V" echo "($datetime) alert sent" >> ${logacv} battstatus=0 else errtime=$(expr `date +%s` - $(stat -c %Y ${logacv})) [ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv} if [ $voltstatus -eq 0 ]; then - SENDNOTICE "Power Nominal" "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V" + SENDNOTICE "Power Nominal" "($datetime) Service Startup +Normal voltage detected +VOLTAGE: ${INPUTACV}V" echo "($datetime) - ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv} fi fi @@ -202,7 +214,9 @@ CHECKACV_SERVICE(){ else echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv} if [ $voltstatus -lt 3 ]; then - SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V\nBATT VOLTAGE: ${BATTVOLT}V" 1 + SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected +VOLTAGE: ${INPUTACV}V +BATT VOLTAGE: ${BATTVOLT}V" 1 echo "($datetime) alert sent" >> ${logacv} last_battvolt=$BATTVOLT fi @@ -211,7 +225,8 @@ CHECKACV_SERVICE(){ echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv} if [ $battstatus -lt 2 ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" 1 + SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V +Shutting down main servers" 1 # SHUTDOWN_SERVERS MAIN battstatus=2 fi @@ -220,9 +235,10 @@ CHECKACV_SERVICE(){ echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv} if [ $battstatus -lt 3 ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv} - SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" 1 + SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V +Shutting down all servers" 1 # SHUTDOWN_SERVERS SYS - curl --data "cmd=PWRoff" http://10.10.0.61/cmd + # curl --data "cmd=PWRoff" http://10.10.0.61/cmd battstatus=3 fi @@ -380,10 +396,10 @@ fi checktemp_service) if [ "${2}" = "stop" ]; then - STOP_SERVICE ${3} - echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${FOLDER}/log-${3} + STOP_SERVICE + echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${FOLDER}/log-temp else - CHECKTEMP_SERVICE ${3} + CHECKTEMP_SERVICE fi ;; checkacv_service)