#!/usr/bin/env bash # powerwall - CLI commands to control VM guest power action="$1" FOLDER='/opt/idssys/powerwall' source $FOLDER/defaults.inc source $FOLDER/powerwall.conf source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc shopt -s lastpipe VERBOSE=true #ssh root@10.5.10.35 '/vmfs/volumes/NFS_ESXi-Vault/esxi-shutdown.sh > /dev/null 2>&1' & > /dev/null 2>&1 CHECKTEMP(){ mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` #echo "$temp_c -> $temp_f" echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then echo "WARNING: $temp_f°F" elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then echo "CRITICAL: $temp_f°F" else echo "normal: $temp_f°F" fi fi } CHECKTEMP_SERVICE(){ last_temp=0 mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` while true do ${mqtt_conn} -t tele/${1}/SENSOR | while read -r mqtt_message do echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c echo $mqtt_message | cut -d'T' -f 2 | cut -d"\"" -f 3 | read temp_d echo $mqtt_message | cut -d'T' -f 3 | cut -d"\"" -f 1 | read temp_t datetime="${temp_d} ${temp_t}" datetime=$(TZ=UTC0 date -d "${datetime}" +%s) datetime=`date -d @${datetime} +'%Y-%m-%d %H:%M:%S'` if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` #echo "$temp_c -> $temp_f" echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys temp_diff=$((${last_temp%.*} - ${temp_f%.*})) [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then echo "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" | mail -s "${1} TEMP WARNING" $email_alert [ $VERBOSE = true ] && echo "alert sent" last_temp=$temp_f fi [ $VERBOSE = true ] && echo "($datetime) WARNING: ${temp_f}°F - (Difference of ${temp_diff}°)" elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then [ $VERBOSE = true ] && echo "($datetime) CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)" if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then echo -e "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert [ $VERBOSE = true ] && echo "alert sent" last_temp=$temp_f SHUTDOWN_SERVERS MAIN echo -e "($datetime) Main servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert fi elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then echo "($datetime) SYSTEM CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)" if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then echo -e "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert [ $VERBOSE = true ] && echo "alert sent" last_temp=$temp_f SHUTDOWN_SERVERS SYS echo -e "($datetime) System servers have been shutdown\nAll servers are now offline" | mail -s "SERVERS SHUTDOWN" $email_alert fi else if [ $last_temp -gt 1 ]; then echo -e "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F" | mail -s "${1} BACK TO NORMAL" $email_alert [ $VERBOSE = true ] && echo "alert sent" elif [ $last_temp -eq 0 ]; then echo "NORMAL TEMP: ${temp_f}°F" | mail -s "${1} TEMP NORMAL" $email_alert; fi last_temp=1 [ $VERBOSE = true ] && echo "($datetime) normal: ${temp_f}°F" fi QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')" ${mysql_conn} -e "${QRY}" else echo "($datetime) ERROR reading temperature" | mail -s "${1} ERROR" $email_alert [ $VERBOSE = true ] && echo "alert sent" fi done sleep 10 done # & } CHECKACV_SERVICE(){ voltstatus=0 while true do INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV') datetime=`date +'%Y-%m-%d %H:%M:%S'` if [ ${INPUTACV} -ge ${min_volt} ]; then if [ $voltstatus -eq 3 ]; then [ $VERBOSE = true ] && echo "($datetime) back to normal voltage: ${INPUTACV}V" echo -e "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "POWER NOMINAL AGAIN" $email_alert [ $VERBOSE = true ] && echo "alert sent" else [ $VERBOSE = true ] && echo "($datetime) normal voltage: ${INPUTACV}V" if [ $voltstatus -eq 0 ]; then echo -e "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "Power Nominal" $email_alert [ $VERBOSE = true ] && echo "alert sent" fi fi voltstatus=1 else echo "($datetime) LOW voltage: ${INPUTACV}V" if [ $voltstatus -lt 3 ]; then echo -e "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "POWER ALERT - LOW POWER!!" $email_alert [ $VERBOSE = true ] && echo "alert sent" fi voltstatus=3 fi sleep 10 done # & } SHUTDOWN_SERVERS(){ touch $FOLDER/shutdown HOSTS=${1}HOSTS[@] for ESXIHOST in "${!HOSTS}"; do echo "Shutting Down: ${ESXIHOST}" /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" done for ESXIHOST in "${!HOSTS}"; do echo -en "Waiting for Host (${ESXIHOST}) to shutdown... " while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1 done echo "Ok" done if [ "${1}" = "MAIN" ]; then /usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" echo -en "Waiting for iSCSI-Server to shutdown... " while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1 done echo "Ok" fi } if [ ${action-x} ]; then case $action in checktemp) CHECKTEMP ${2};; checktemp_service) if [ "${2}" = "stop" ]; then service_pid=`systemctl show --property MainPID --value ${SERVICE_NAME[${3}]}` /bin/kill -9 $service_pid #/bin/systemctl stop $srvcname #/usr/bin/killall bash else CHECKTEMP_SERVICE ${2} fi ;; checkacv_service) if [ "${2}" = "stop" ]; then service_pid=`systemctl show --property MainPID --value ${SERVICE_NAME[${3}]}` /bin/kill -9 $service_pid #/bin/systemctl stop $srvcname #/usr/bin/killall bash else CHECKACV_SERVICE fi ;; esac fi exit 0