#!/usr/bin/env bash # powerwall - CLI commands to control VM guest power action="${1}" source /opt/idssys/powerwall/defaults.inc source /opt/idssys/powerwall/powerwall.conf source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc shopt -s lastpipe VERBOSE=false CHECKTEMP(){ cw=20; echo echo -e "${idsCL[Yellow]}Sensor Temperature(s) ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}" DIVIDER . yellow 75 if [ "${1}" != "" ]; then # reading=$(CHECKTEMPSENSOR ${1}) # c=0; spc=''; spc1=`expr ${cw} - ${#1}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done # echo -e "${idsCL[LightCyan]}${1}${spc}${idsCL[Default]}: ${reading}" for SENSOR in ${!SENSOR_ID[@]}; do if [[ "${SENSOR}" = *"${1}"* ]]; then reading=$(CHECKTEMPSENSOR ${SENSOR}) c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}" fi done else for SENSOR in ${!SENSOR_ID[@]}; do reading=$(CHECKTEMPSENSOR ${SENSOR}) c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}" done fi echo } CHECKTEMPSENSOR(){ echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit if [ "${SENSOR_TYPE[${1}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${SENSOR_TYPE[${1}]}" == "system" ]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} fi if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` #echo "$temp_c -> $temp_f" if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then echo -en "${idsCL[Yellow]}WARNING ($temp_f°F)${idsCL[Default]}" elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then echo -en "${idsCL[Red]}CRITICAL ($temp_f°F)${idsCL[Default]}" else echo -en "${idsCL[Green]}Normal ($temp_f°F)${idsCL[Default]}" fi c=0; spc=''; spc1=`expr 13 - ${#temp_f} - 2`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done [ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' ' [ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' ' echo -e "${spc}[${spcw}${idsCL[Green]}<=$(expr ${temp_warn} - 1)°F${idsCL[Default]} /${spcw}${idsCL[Yellow]}${temp_warn}°F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}°F${idsCL[Default]} ]" fi } CHECKTEMP_SERVICE(){ cw=20 declare -A last_temp while true; do for SENSOR in ${!SENSOR_ID[@]}; do if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' fi datetime=`date +'%Y-%m-%d %H:%M:%S'` if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys if [ "${last_temp[${SENSOR}]}" != "" ]; then temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}` [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) else last_temp[${SENSOR}]=0 temp_diff=0 fi if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}°F - (Difference of ${temp_diff}°)" 1 echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} last_temp[${SENSOR}]=$temp_f fi echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}°)" >> ${logtemp} elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then echo "($datetime) - ${temp_f} F - CRITICAL - (Difference of ${temp_diff} F)" >> ${logtemp} if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°) Shutting down servers!!" 1 echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} last_temp[${SENSOR}]=$temp_f # SHUTDOWN_SERVERS MAIN SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1 echo -e "($datetime) - ${SENSOR} - ${temp_f}F - Main servers have been shutdown" >> ${logtemp} fi elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}°F - (Difference of ${temp_diff}°) Shutting down system servers!!" 1 echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} last_temp[${SENSOR}]=$temp_f # SHUTDOWN_SERVERS SYS SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1 echo -e "($datetime) - ${SENSOR} - ${temp_f}F - System servers have been shutdown" >> ${logtemp} fi else if [ ${last_temp[${SENSOR}]} -gt 1 ]; then SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}°F\nPrevious Temp: ${last_temp[${SENSOR}]}°F" echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp} elif [ $last_temp -eq 0 ]; then SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup\n NORMAL TEMP: ${temp_f}°F" echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} fi last_temp[${SENSOR}]=1 errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp})) [ $errtime -ge 3600 ] && echo "($datetime) - ${SENSOR} - ${temp_f}F - Normal Temp" >> ${logtemp} fi if [ "${temp_h}" != "" ]; then QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" else QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" fi ${mysql_conn} -e "${QRY}" else SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1 echo "($datetime) - ${SENSOR} - alert sent" >> ${logtemp} fi done sleep 10 done # & } CHECKPOWER(){ JSONSTATS=$(curl -s "http://10.10.0.61/stats.json") INPUTV=`echo ${JSONSTATS} | jq '.inputs .inV'` INPUTA=`echo ${JSONSTATS} | jq '.inputs .inA'` OUTPUTV=`echo ${JSONSTATS} | jq '.outputs .outV'` OUTPUTA=`echo ${JSONSTATS} | jq '.outputs .outA'` BATTV=`echo ${JSONSTATS} | jq '.inputs .battV'` BATTA=`echo ${JSONSTATS} | jq '.inputs .xfA'` OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}') OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}') echo echo -e "${idsCL[Yellow]}Power Information" DIVIDER . yellow 75 echo -e "${idsCL[LightCyan]}Input Power ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}V / ${INPUTA}A" echo -e "${idsCL[LightCyan]}Output Power ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)" echo -e "${idsCL[LightCyan]}Battery Power ${idsCL[Default]}: ${idsCL[Green]}${BATTV}V / ${BATTA}A ${idsCL[Default]}" [ "${action}" != "" ] && echo } CHECKACV_SERVICE(){ voltstatus=0 battstatus=0 while true; do INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV') BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV') datetime=`date +'%Y-%m-%d %H:%M:%S'` if [ ${INPUTACV} -ge ${min_acvolt} ]; then if [ $voltstatus -eq 3 ]; then echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv} SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected VOLTAGE: ${INPUTACV}V" echo "($datetime) alert sent" >> ${logacv} battstatus=0 else errtime=$(expr `date +%s` - $(stat -c %Y ${logacv})) [ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv} if [ $voltstatus -eq 0 ]; then SENDNOTICE "Power Nominal" "($datetime) Service Startup Normal voltage detected VOLTAGE: ${INPUTACV}V" echo "($datetime) - ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv} fi fi voltstatus=1 else echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv} if [ $voltstatus -lt 3 ]; then SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected VOLTAGE: ${INPUTACV}V BATT VOLTAGE: ${BATTVOLT}V" 1 echo "($datetime) alert sent" >> ${logacv} last_battvolt=$BATTVOLT fi voltstatus=3 if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv} if [ $battstatus -lt 2 ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv} SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V Shutting down main servers" 1 # SHUTDOWN_SERVERS MAIN battstatus=2 fi elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv} if [ $battstatus -lt 3 ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv} SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V Shutting down all servers" 1 # SHUTDOWN_SERVERS SYS # curl --data "cmd=PWRoff" http://10.10.0.61/cmd battstatus=3 fi else errtime=$(expr `date +%s` - $(stat -c %Y ${logacv})) [ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logacv} battstatus=1 fi volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*})) [ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1)) if [ $volt_diff -gt 0 ]; then SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V" echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv} last_battvolt=$BATTVOLT fi fi sleep 10 done # & } POWERLOGGER_SERVICE(){ while true; do datetime=`date +'%Y-%m-%d %H:%M:%S'` JSONSTATS=$(curl -s "http://10.10.0.61/stats.json") inV=`echo ${JSONSTATS} | jq '.inputs .inV'` inA=`echo ${JSONSTATS} | jq '.inputs .inA'` outV=`echo ${JSONSTATS} | jq '.outputs .outV'` outA=`echo ${JSONSTATS} | jq '.outputs .outA'` battV=`echo ${JSONSTATS} | jq '.inputs .battV'` xfA=`echo ${JSONSTATS} | jq '.inputs .xfA'` QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" sleep 1m done } RESTART_SERVICES(){ STOPALL_SERVICES sleep 5s STARTALL_SERVICES } STOPALL_SERVICES(){ for srvc in "${POWERWALL_SERVICES[@]}"; do echo "Stopping $srvc" /bin/systemctl stop $srvc done } STARTALL_SERVICES(){ for srvc in "${POWERWALL_SERVICES[@]}"; do echo "Starting $srvc" /bin/systemctl start $srvc done } STOP_SERVICE(){ echo "stop: ${1}" service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}` /usr/bin/pkill -P $service_pid } SHUTDOWN_SERVERS(){ touch $FOLDER/shutdown HOSTS=${1}HOSTS[@] for ESXIHOST in "${!HOSTS}"; do echo "Shutting Down: ${ESXIHOST}" /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" done for ESXIHOST in "${!HOSTS}"; do echo -en "Waiting for Host (${ESXIHOST}) to shutdown... " while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1 done echo "Ok" done if [ "${1}" = "MAIN" ]; then /opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 10.10.2.17 >/dev/null 2>&1 /usr/bin/ssh root@10.10.2.10 "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh" echo -en "Waiting for iSCSI-Server to shutdown... " while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1 done echo "Ok" fi } SHUTDOWN_SERVER(){ echo "Shutting Down: ${1}" /usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh" /usr/bin/ssh root@${1} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${1}.sh" /usr/bin/ssh root@${1} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${1}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh" /usr/bin/ssh root@${1} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${1}.sh" echo -en "Waiting for Host (${1}) to shutdown... " while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do sleep 1 done echo "Ok" } ISCSIVMSHUTDOWN(){ for ESXIHOST in "${ESXIHOSTS[@]}"; do echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... " # /opt/idssys/powerwall/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 echo -e "${idsCL[Green]}DONE${idsCL[Default]}" echo done } TEST(){ echo "Getting info from ${1}" /opt/idssys/powerwall/esxi-scripts/sys-iscsi-shutdown.ps1 ${1} } if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then echo DIVIDER . lightGreen 50 c=0; spc=''; spc1=`expr 18 - ${#VERS}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -e "${idsCL[LightGreen]}| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${spc}${idsCL[LightGreen]}|${idsCL[Default]}" DIVIDER . lightGreen 50 fi # if [ ${action-x} ]; then case $action in stopall) STOPALL_SERVICES;; startall) STARTALL_SERVICES;; restartall) RESTART_SERVICES;; checktemp) CHECKTEMP ${2};; checkpower) CHECKPOWER;; test) TEST ${2};; iscsi-vm-shutdown) ISCSIVMSHUTDOWN;; shutdownhost) SHUTDOWN_SERVER ${2};; checktemp_service) if [ "${2}" = "stop" ]; then STOP_SERVICE echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${FOLDER}/log-temp else CHECKTEMP_SERVICE fi ;; checkacv_service) if [ "${2}" = "stop" ]; then STOP_SERVICE ACV echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logacv} else CHECKACV_SERVICE fi ;; powerlogger_service) if [ "${2}" = "stop" ]; then STOP_SERVICE powerlogger else POWERLOGGER_SERVICE fi ;; update);; *) CHECKPOWER CHECKTEMP ;; esac # fi exit 0