360 lines
14 KiB
Bash
Executable File
360 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
action="$1"
|
|
|
|
source /opt/idssys/powerwall/defaults.inc
|
|
source /opt/idssys/powerwall/powerwall.conf
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
|
|
shopt -s lastpipe
|
|
|
|
VERBOSE=false
|
|
|
|
|
|
#ssh root@10.5.10.35 '/vmfs/volumes/NFS_ESXi-Vault/esxi-shutdown.sh > /dev/null 2>&1' & > /dev/null 2>&1
|
|
|
|
CHECKTEMP(){
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
#echo "$temp_c -> $temp_f"
|
|
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
echo "WARNING: $temp_f°F"
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
|
|
echo "CRITICAL: $temp_f°F"
|
|
else
|
|
echo "normal: $temp_f°F"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
CHECKTEMP_SERVICE(){
|
|
logsrvc=${FOLDER}/log-${1}
|
|
[ ! -f ${logsrvc} ] && touch ${logsrvc}
|
|
last_temp=0
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
while true
|
|
do
|
|
${mqtt_conn} -t tele/${1}/SENSOR | while read -r mqtt_message
|
|
do
|
|
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
echo $mqtt_message | cut -d'T' -f 2 | cut -d"\"" -f 3 | read temp_d
|
|
echo $mqtt_message | cut -d'T' -f 3 | cut -d"\"" -f 1 | read temp_t
|
|
# datetime="${temp_d} ${temp_t}"
|
|
# datetime=$(TZ=UTC0 date -d "${datetime}" +%s)
|
|
# datetime=`date -d @${datetime} +'%Y-%m-%d %H:%M:%S'`
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
|
|
#echo "$temp_c -> $temp_f"
|
|
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 3 | read temp_crit_sys
|
|
|
|
temp_diff=$((${last_temp%.*} - ${temp_f%.*}))
|
|
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
SENDNOTICE "${1} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1
|
|
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
|
|
last_temp=$temp_f
|
|
fi
|
|
echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}°)" >> ${logsrvc}
|
|
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
|
|
echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logsrvc}
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
SENDNOTICE "${1} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down servers!!" 1
|
|
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
|
|
last_temp=$temp_f
|
|
# SHUTDOWN_SERVERS MAIN
|
|
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
|
|
echo -e "($datetime) - ${temp_f}F - Main servers have been shutdown" >> ${logsrvc}
|
|
|
|
fi
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
|
|
echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logsrvc}
|
|
if [ $temp_diff -gt 1 ] || [ "$last_temp" = "1" ]; then
|
|
SENDNOTICE "${1} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)\nShutting down system servers!!" 1
|
|
echo "($datetime) - ${temp_f}F - alert sent" >> ${logsrvc}
|
|
last_temp=$temp_f
|
|
# SHUTDOWN_SERVERS SYS
|
|
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
|
|
echo -e "($datetime) - ${temp_f}F - System servers have been shutdown" >> ${logsrvc}
|
|
fi
|
|
else
|
|
if [ $last_temp -gt 1 ]; then
|
|
SENDNOTICE "${1} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp}°F"
|
|
echo -e "($datetime) - ${temp_f}F - Back to NORMAL TEMP - Previous Temp: ${last_temp}F" >> ${logsrvc}
|
|
echo "($datetime) alert sent" >> ${logsrvc}
|
|
elif [ $last_temp -eq 0 ]; then
|
|
SENDNOTICE "${1} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F"
|
|
echo -e "($datetime) - ${temp_f}F - Service Startup - NORMAL TEMP" >> ${logsrvc}
|
|
fi
|
|
last_temp=1
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logsrvc}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${temp_f}F - Normal Temp" >> ${logsrvc}
|
|
fi
|
|
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[$1]}','${datetime}','${temp_f}','${temp_h}')"
|
|
${mysql_conn} -e "${QRY}"
|
|
else
|
|
SENDNOTICE "${1} ERROR" "($datetime) ERROR reading temperature" 1
|
|
echo "($datetime) alert sent" >> ${logsrvc}
|
|
fi
|
|
done
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
CHECKPOWER(){
|
|
JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
|
|
INPUTV=`echo ${JSONSTATS} | jq '.inputs .inV'`
|
|
INPUTA=`echo ${JSONSTATS} | jq '.inputs .inA'`
|
|
OUTPUTV=`echo ${JSONSTATS} | jq '.outputs .outV'`
|
|
OUTPUTA=`echo ${JSONSTATS} | jq '.outputs .outA'`
|
|
BATTV=`echo ${JSONSTATS} | jq '.inputs .battV'`
|
|
BATTA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
|
|
|
|
OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
|
|
OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
|
|
|
|
echo "Input Power: ${INPUTV}V / ${INPUTA}A"
|
|
echo "Output Power: ${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
|
|
echo "Battery Power: ${BATTV}V / ${BATTA}A"
|
|
}
|
|
|
|
CHECKACV_SERVICE(){
|
|
voltstatus=0
|
|
battstatus=0
|
|
while true; do
|
|
INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV')
|
|
BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV')
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
|
|
if [ $voltstatus -eq 3 ]; then
|
|
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv}
|
|
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected\nVOLTAGE: ${INPUTACV}V"
|
|
echo "($datetime) alert sent" >> ${logacv}
|
|
battstatus=0
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv}
|
|
if [ $voltstatus -eq 0 ]; then
|
|
SENDNOTICE "Power Nominal" "($datetime) Service Startup\nNormal voltage detected\nVOLTAGE: ${INPUTACV}V"
|
|
echo "($datetime) - ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
|
|
fi
|
|
fi
|
|
voltstatus=1
|
|
else
|
|
echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
|
|
if [ $voltstatus -lt 3 ]; then
|
|
SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected\nVOLTAGE: ${INPUTACV}V\nBATT VOLTAGE: ${BATTVOLT}V" 1
|
|
echo "($datetime) alert sent" >> ${logacv}
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
voltstatus=3
|
|
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
|
|
if [ $battstatus -lt 2 ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
|
|
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V\nShutting down main servers" 1
|
|
# SHUTDOWN_SERVERS MAIN
|
|
battstatus=2
|
|
fi
|
|
|
|
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
|
|
if [ $battstatus -lt 3 ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
|
|
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V\nShutting down all servers" 1
|
|
# SHUTDOWN_SERVERS SYS
|
|
curl --data "cmd=PWRoff" http://10.10.0.61/cmd
|
|
|
|
battstatus=3
|
|
fi
|
|
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logacv}
|
|
|
|
battstatus=1
|
|
|
|
fi
|
|
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
|
|
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
|
|
if [ $volt_diff -gt 0 ]; then
|
|
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
|
|
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv}
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
fi
|
|
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
POWERLOGGER_SERVICE(){
|
|
while true
|
|
do
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
|
|
inV=`echo ${JSONSTATS} | jq '.inputs .inV'`
|
|
inA=`echo ${JSONSTATS} | jq '.inputs .inA'`
|
|
outV=`echo ${JSONSTATS} | jq '.outputs .outV'`
|
|
outA=`echo ${JSONSTATS} | jq '.outputs .outA'`
|
|
battV=`echo ${JSONSTATS} | jq '.inputs .battV'`
|
|
xfA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
sleep 1m
|
|
|
|
done
|
|
}
|
|
|
|
RESTART_SERVICES(){
|
|
STOPALL_SERVICES
|
|
sleep 5s
|
|
STARTALL_SERVICES
|
|
}
|
|
STOPALL_SERVICES(){
|
|
for srvc in "${POWERWALL_SERVICES[@]}"; do
|
|
echo "Stopping $srvc"
|
|
/bin/systemctl stop $srvc
|
|
done
|
|
}
|
|
STARTALL_SERVICES(){
|
|
for srvc in "${POWERWALL_SERVICES[@]}"; do
|
|
echo "Starting $srvc"
|
|
/bin/systemctl start $srvc
|
|
done
|
|
}
|
|
|
|
STOP_SERVICE(){
|
|
echo "stop: ${1}"
|
|
service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}`
|
|
/usr/bin/pkill -P $service_pid
|
|
}
|
|
|
|
SHUTDOWN_SERVERS(){
|
|
touch $FOLDER/shutdown
|
|
|
|
HOSTS=${1}HOSTS[@]
|
|
for ESXIHOST in "${!HOSTS}"; do
|
|
echo "Shutting Down: ${ESXIHOST}"
|
|
/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
/usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
done
|
|
for ESXIHOST in "${!HOSTS}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Ok"
|
|
done
|
|
|
|
if [ "${1}" = "MAIN" ]; then
|
|
/opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 10.10.2.17 >/dev/null 2>&1
|
|
|
|
/usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
echo -en "Waiting for iSCSI-Server to shutdown... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Ok"
|
|
fi
|
|
}
|
|
SHUTDOWN_SERVER(){
|
|
echo "Shutting Down: ${1}"
|
|
/usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
|
|
/usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${1}.sh"
|
|
/usr/bin/ssh root@${1} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${1}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
|
|
/usr/bin/ssh root@${1} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh"
|
|
|
|
echo -en "Waiting for Host (${1}) to shutdown... "
|
|
while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Ok"
|
|
}
|
|
|
|
ISCSIVMSHUTDOWN(){
|
|
for ESXIHOST in "${ESXIHOSTS[@]}"; do
|
|
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... "
|
|
# /opt/idssys/powerwall/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}DONE${idsCL[Default]}"
|
|
echo
|
|
done
|
|
|
|
}
|
|
|
|
TEST(){
|
|
echo "Getting info from ${1}"
|
|
/opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 ${1}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if [ ${action-x} ]; then
|
|
case $action in
|
|
stopall) STOPALL_SERVICES;;
|
|
startall) STARTALL_SERVICES;;
|
|
restartall) RESTART_SERVICES;;
|
|
checktemp) CHECKTEMP ${2};;
|
|
checkpower) CHECKPOWER;;
|
|
test) TEST ${2};;
|
|
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
|
|
|
|
shutdownhost) SHUTDOWN_SERVER ${2};;
|
|
|
|
checktemp_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE ${3}
|
|
else
|
|
CHECKTEMP_SERVICE ${3}
|
|
fi
|
|
;;
|
|
checkacv_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE ACV
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logacv}
|
|
else
|
|
CHECKACV_SERVICE
|
|
fi
|
|
;;
|
|
powerlogger_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE powerlogger
|
|
else
|
|
POWERLOGGER_SERVICE
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
exit 0 |