241 lines
9.3 KiB
Bash
Executable File
241 lines
9.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
action="$1"
|
|
FOLDER='/opt/idssys/powerwall'
|
|
|
|
source $FOLDER/defaults.inc
|
|
source $FOLDER/powerwall.conf
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
|
|
shopt -s lastpipe
|
|
|
|
VERBOSE=true
|
|
|
|
|
|
#ssh root@10.5.10.35 '/vmfs/volumes/NFS_ESXi-Vault/esxi-shutdown.sh > /dev/null 2>&1' & > /dev/null 2>&1
|
|
|
|
CHECKTEMP(){
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
#echo "$temp_c -> $temp_f"
|
|
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
echo "WARNING: $temp_f°F"
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
|
|
echo "CRITICAL: $temp_f°F"
|
|
else
|
|
echo "normal: $temp_f°F"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
CHECKTEMP_SERVICE(){
|
|
last_temp=0
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
while true
|
|
do
|
|
${mqtt_conn} -t tele/${1}/SENSOR | while read -r mqtt_message
|
|
do
|
|
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
echo $mqtt_message | cut -d'T' -f 2 | cut -d"\"" -f 3 | read temp_d
|
|
echo $mqtt_message | cut -d'T' -f 3 | cut -d"\"" -f 1 | read temp_t
|
|
datetime="${temp_d} ${temp_t}"
|
|
datetime=$(TZ=UTC0 date -d "${datetime}" +%s)
|
|
datetime=`date -d @${datetime} +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
|
|
#echo "$temp_c -> $temp_f"
|
|
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys
|
|
|
|
temp_diff=$((${last_temp%.*} - ${temp_f%.*}))
|
|
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
echo "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" | mail -s "${1} TEMP WARNING" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
last_temp=$temp_f
|
|
fi
|
|
[ $VERBOSE = true ] && echo "($datetime) WARNING: ${temp_f}°F - (Difference of ${temp_diff}°)"
|
|
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)"
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
echo -e "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
last_temp=$temp_f
|
|
SHUTDOWN_SERVERS MAIN
|
|
echo -e "($datetime) Main servers have been shutdown" | mail -s "SERVERS SHUTDOWN" $email_alert
|
|
|
|
fi
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
|
|
echo "($datetime) SYSTEM CRITICAL: ${temp_f}°F - (Difference of ${temp_diff}°)"
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
echo -e "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" | mail -s "${1} TEMP CRITICAL" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
last_temp=$temp_f
|
|
SHUTDOWN_SERVERS SYS
|
|
echo -e "($datetime) System servers have been shutdown\nAll servers are now offline" | mail -s "SERVERS SHUTDOWN" $email_alert
|
|
fi
|
|
else
|
|
if [ $last_temp -gt 1 ]; then
|
|
echo -e "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F" | mail -s "${1} BACK TO NORMAL" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
elif [ $last_temp -eq 0 ]; then
|
|
echo -e "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F" | mail -s "${1} TEMP NORMAL" $email_alert;
|
|
fi
|
|
last_temp=1
|
|
[ $VERBOSE = true ] && echo "($datetime) normal: ${temp_f}°F"
|
|
fi
|
|
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')"
|
|
${mysql_conn} -e "${QRY}"
|
|
else
|
|
echo "($datetime) ERROR reading temperature" | mail -s "${1} ERROR" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
fi
|
|
done
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
CHECKACV_SERVICE(){
|
|
voltstatus=0
|
|
battstatus=0
|
|
while true
|
|
do
|
|
INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV')
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
|
|
if [ $voltstatus -eq 3 ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) back to normal voltage: ${INPUTACV}V"
|
|
echo -e "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "POWER NOMINAL AGAIN" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
battstatus=0
|
|
else
|
|
[ $VERBOSE = true ] && echo "($datetime) normal voltage: ${INPUTACV}V"
|
|
if [ $voltstatus -eq 0 ]; then
|
|
echo -e "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "Power Nominal" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
fi
|
|
fi
|
|
voltstatus=1
|
|
else
|
|
[ $VERBOSE = true ] && echo "($datetime) LOW voltage: ${INPUTACV}V"
|
|
if [ $voltstatus -lt 3 ]; then
|
|
echo -e "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V" | mail -s "POWER ALERT - LOW POWER!!" $email_alert
|
|
[ $VERBOSE = true ] && echo "($datetime) alert sent"
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
voltstatus=3
|
|
BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV')
|
|
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) LOW BATTERY voltage: ${BATTVOLT}V"
|
|
if [ $battstatus -lt 2 ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) Shutting down main servers..."
|
|
echo -e "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" | mail -s "BATT-VOLT LOW: SHUTDOWN SERVERS" $email_alert
|
|
|
|
battstatus=2
|
|
fi
|
|
|
|
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) SYSTEM LOW BATTERY voltage: ${BATTVOLT}V"
|
|
if [ $battstatus -lt 3 ]; then
|
|
[ $VERBOSE = true ] && echo "($datetime) Shutting down all remaining servers..."
|
|
echo -e "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" | mail -s "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" $email_alert
|
|
|
|
battstatus=3
|
|
fi
|
|
|
|
else
|
|
[ $VERBOSE = true ] && echo "($datetime) Battery Voltage: ${BATTVOLT}V"
|
|
|
|
battstatus=1
|
|
|
|
fi
|
|
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
|
|
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
|
|
if [ $volt_diff -gt 0 ]; then
|
|
echo -e "($datetime) Battery Voltage: ${BATTVOLT}V" | mail -s "BATTERY VOLTAGE CHANGE" $email_alert
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
fi
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
SHUTDOWN_SERVERS(){
|
|
touch $FOLDER/shutdown
|
|
|
|
HOSTS=${1}HOSTS[@]
|
|
for ESXIHOST in "${!HOSTS}"; do
|
|
echo "Shutting Down: ${ESXIHOST}"
|
|
/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
done
|
|
for ESXIHOST in "${!HOSTS}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Ok"
|
|
done
|
|
|
|
if [ "${1}" = "MAIN" ]; then
|
|
/usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
echo -en "Waiting for iSCSI-Server to shutdown... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Ok"
|
|
fi
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
if [ ${action-x} ]; then
|
|
case $action in
|
|
checktemp) CHECKTEMP ${2};;
|
|
checktemp_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
service_pid=`systemctl show --property MainPID --value ${SERVICE_NAME[${3}]}`
|
|
/usr/bin/pkill -P $service_pid
|
|
#/bin/systemctl stop $srvcname
|
|
#/usr/bin/killall bash
|
|
|
|
else
|
|
CHECKTEMP_SERVICE ${2}
|
|
fi
|
|
;;
|
|
checkacv_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
service_pid=`systemctl show --property MainPID --value ${SERVICE_NAME['ACV']}`
|
|
/usr/bin/pkill -P $service_pid
|
|
#/bin/systemctl stop $srvcname
|
|
#/usr/bin/killall bash
|
|
|
|
else
|
|
CHECKACV_SERVICE
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
exit 0 |