voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
00adae1aa7fd40caec3644aade22bedc4acecc00
powerwall/powerwall.sh
David Schroeder 00adae1aa7 Update powerwall.sh
2023-11-29 21:44:16 -06:00

1731 lines
78 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power
action="${1}"
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/powerwall/defaults.inc
source ${PW_FOLDER}/settings.conf
# TEST=true
shopt -s lastpipe
VERBOSE=false
CHECKTEMP(){
start=`date +%s`
cw=25;
while [ $# -gt 0 ]; do
case "$1" in
-avg) avgdays=${2};;
-search) search=${2};;
esac
shift 1
done
echo
if [ "${avgdays}" != "" ]; then
[ ${avgdays} -lt 10 ] && avgdays_disp=" ${avgdays}" || avgdays_disp=${avgdays}
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current / ${avgdays_disp}DayAVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[Red]}critical${idsCL[Default]}"
else
avgdays=noavg
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[Red]}critical${idsCL[Default]}"
fi
DIVIDER . yellow 90
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]; then
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ]; then
vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]${idsCL[Default]}"
echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
IFS=$'\n'
read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}"
unset IFS
echo -en "\r\033[K"
if [ ${#vHOST_TEMPNAMES[@]} > 1 ]; then
t=1
for sensorname in "${vHOST_TEMPNAMES[@]}"; do
temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[LightYellow]}Calculating ${avgdays} Day average ... "
reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays})
[ "${avgdays}" != "noavg" ] && echo -en "\r\033[K"
echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}"
((t++))
done
else
echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data"
fi
else #host down
echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down"
# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
# SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
# touch ${PW_TMPFOLDER}/${EHOST}.down
# fi
fi
else
[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ... " || echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays})
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
fi
echo
fi
done
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
[ "${action}" != "" ] && echo
}
CHECKTEMPSENSOR(){
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
if [ "${6}" != "noavg" ]; then
[ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3
if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays})
else
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5})
fi
else
average=''
fi
if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
if [ "${temp_c}" != "" ]; then
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
fi
elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
temp_c=${2}
temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`
fi
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
#echo "${temp_c} -> ${temp_f}"
c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
if [ "${average}" != "" ]; then
c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
if [ "${average}" == "" ]; then
averagedisp="${idsCL[Green]}"
elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
averagedisp="${idsCL[Yellow]}${average}'F"
elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
averagedisp="${idsCL[Red]}${average}'F"
else
averagedisp="${idsCL[Green]}${average}'F"
fi
fi
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
[ "${average}" != "" ] && echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}'F${idsCL[Default]}"
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then
[ "${average}" != "" ] && echo -en "${idsCL[Red]}CRITICAL${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Red]}CRITICAL${spct}${temp_f}'F${idsCL[Default]}"
else
[ "${average}" != "" ] && echo -en "${idsCL[Green]}Normal ${spct}${temp_f}'F /${spca}${averagedisp}${idsCL[Default]}" || echo -en "${idsCL[Green]}Normal ${spct}${temp_f}'F${idsCL[Default]}"
fi
tmp="${spct}${temp_f}"
c=0; spc=''; spc1=`expr 10 - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F=>${idsCL[Default]} ]"
fi
}
AVERAGETEMP(){
temptotal=0
daysback=${2}
sensorid=${1}
logsfound=0;
if [ "${3}" != "" ]; then
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
entityid=${entityid[1]}
WHERE="sensorid=${sensorid} AND entity=${entityid}"
else
WHERE="sensorid=${sensorid}"
fi
while IFS=$'\t' read date temp hum ;do
temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
echo ${average}
}
SENDDAILY(){
dailtemp=$(DAILYTEMP)
SENDNOTICE "Daily Temp Readings" "${dailtemp}"
}
CHECKTEMP_SERVICE(){
cw=20
declare -A last_temp
while true; do
# start=`date +%s`
# time="$(date +%H)$(date +%M)"
# if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
# if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
# touch ${PW_TMPFOLDER}/.sentdaily
# SENDDAILY &
# fi
# fi
for SENSORa in ${PW_SENSOR_ORD[@]}; do
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
[ ! -f ${logtemp} ] && touch ${logtemp}
[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge 3600 ] && relog=1 || relog=0
if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
fi
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
IFS=$'\n'
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
unset IFS
[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
t=1
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq)
# FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
if [ ${FanSpeed} -gt 92 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30%25" 1
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
FanSpeed=30
fi
fi
else
SENSORS_CHECK=(${SENSORa})
fi
if [ "${SENSORS_CHECK}" != "" ]; then
for SENSOR in "${SENSORS_CHECK[@]}"; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
temp_crit_sys=${temp_crit}
temp_h=''
vSENSOR=${SENSOR}
SENSOR="${SENSORa}-${vSENSOR// /_}"
((t++))
fi
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
temp_f=$(echo "scale=2; ${temp_c}*1.8 + 32" | bc)
# echo "HERE: ${SENSOR} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
fi
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`echo "scale=2; ${last_temp[${SENSOR}]}-${temp_f}" | bc`
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
last_temp[${SENSOR}]=${temp_f}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
fi
fi
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
SHUTDOWN_SERVER ${SENSORa} &
touch ${PW_TMPFOLDER}/power.${SENSORa}.off
fi
fi
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ $(bc -l <<< "${temp_diff} > 1") -eq 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=${temp_f}
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
fi
else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
# NORMAL TEMP: ${temp_f}'F"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${relog} -eq 1 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${PW_TMPFOLDER}/temp.*
rm -f ${PW_TMPFOLDER}/power.${SENSORa}.off
last_temp[${SENSOR}]=1
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq)
# FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
if [ ${FanSpeed} -ne 30 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
fi
fi
fi
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
if [ "${entityid[1]}" == "" ]; then
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
fi
entityid=${entityid[1]}
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ]; then
QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
${mysql_conn} -e "${QRY2}"
fi
elif [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${PW_TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
fi
done
fi
elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSORa}.down
fi
fi
done
# end=`date +%s`; runtime=$((end-start)); echo "runtime: ${runtime}, pausing for 60secs..."
sleep 60
done # &
}
DAILYTEMP(){
cw=35;
echo
echo -e "Sensor Temperature(s) Now / 1DayAVG / 7DayAVG"
DIVIDER . . 60
for SENSORa in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
IFS=$'\n'
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
unset IFS
t=1
else
SENSORS_CHECK=(${SENSORa})
fi
for SENSOR in "${SENSORS_CHECK[@]}"; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
temp_c=$(echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1)
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
vSENSOR=${SENSOR}
SENSOR="${SENSORa}-${vSENSOR// /_}"
((t++))
else
temp_c=''
fi
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
else
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
fi
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
temp_f=`echo "scale=2; ${temp_c}*1.8 + 32" | bc`
echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
else
echo -e "${SENSOR}${spc}No Data"
fi
done
[ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ] && echo
done
}
AVERAGEPOWER(){
watttotal=0
sensorid=${1}
daysback=${2}
logsfound=0;
while IFS=$'\t' read date watt ;do
watttotal=$(bc <<< "scale=2; ${watttotal}+${watt}")
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,watt from power_data WHERE sensorid=${sensorid} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=2; ${watttotal}/${logsfound}")
# echo "$watttotal-$logsfound=${average}"
echo ${average}
}
CHECKPOWER(){
start=`date +%s`
rm -f ${PW_TMPFOLDER}/*.hostpower*
while [ $# -gt 0 ]; do
case "$1" in
-avg) avgdays=${2};;
-offsite) PTYPE=offsite;;
-mainsite) PTYPE=mainsite;;
esac
shift 1
done
[ "${avgdays}" = "0" ] && unset avgdays
[ "${avgdays}" != "" ] && avgdayshdr="${avgdays}-Day-Avg" || avgdayshdr=""
if [ "${PTYPE}" == "" ] || [ "${PTYPE}" == "all" ] || [ "${PTYPE}" == "mainsite" ]; then
echo
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Mainsite Power Information ${avgdayshdr}"
DIVIDER . yellow 75
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
[[ "${OUTPUTW}" == *"."* ]] && [[ "${OUTPUTW}" != *".0"* ]] && OUTPUTW=`printf "%'.2f\n" ${OUTPUTW}` || OUTPUTW=`printf "%'.0f\n" ${OUTPUTW}`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
BATTLIFE=`echo "scale=2; (${BATTV}*100)/54.6" | bc`
if (( $(bc <<<"${BATTV} > 54.8") )); then
if (( $(bc <<<"${BATTV} > 58") )); then
BATTA_disp="${idsCL[Green]}${BATTA}'amps ${idsCL[LightCyan]}- Bulk Charging"
else
BATTA_disp="${idsCL[Green]}${BATTA}'amps ${idsCL[LightCyan]}- Float Charging"
fi
elif [ "${BATTA}" == "-0.1" ]; then
BATTA_disp="${idsCL[LightCyan]}${BATTA}'amps ${idsCL[LightCyan]}- Standby"
else
BATTA_disp="${idsCL[LightRed]}${BATTA}'amps ${idsCL[LightCyan]}- Discharging"
fi
fi
echo -e "${idsCL[White]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}'volts"
echo -e "${idsCL[White]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}${INPUTA}'amps"
echo
echo -e "${idsCL[White]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}'volts / ${OUTPUTV2}'volts"
echo -e "${idsCL[White]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTA}'amps / ${OUTPUTA2}'amps"
echo -e "${idsCL[White]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTW}'watts"
echo
echo -e "${idsCL[White]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}${BATTV}'volts ~ ${BATTLIFE}%${idsCL[Default]}"
echo -e "${idsCL[White]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
echo
# if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLT=`echo "scale=2; ${SVRRM_VOLT}/10" | bc`
SVRRM_WATT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
[[ "${SVRRM_WATT}" == *"."* ]] && [[ "${SVRRM_WATT}" != *".0"* ]] && SVRRM_WATT_disp=`printf "%'.2f\n" ${SVRRM_WATT}` || SVRRM_WATT_disp=`printf "%'.0f\n" ${SVRRM_WATT}`
SVRRM_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
# fi
#
# if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc`
APCPDU_WATT2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT2_disp=`printf "%'.0f\n" ${APCPDU_WATT2}`
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
[[ "${APCPDU_WATT}" == *"."* ]] && [[ "${APCPDU_WATT}" != *".0"* ]] && APCPDU_WATT_disp=`printf "%'.2f\n" ${APCPDU_WATT}` || APCPDU_WATT_disp=`printf "%'.0f\n" ${APCPDU_WATT}`
# fi
echo -e "${idsCL[White]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_AMP}'amps${idsCL[Default]}"
if [ "${avgdays}" != "" ]; then
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['server-rack']} ${avgdays})
[[ "${average}" == *"."* ]] && [[ "${average}" != *".0"* ]] && average=`printf "%'.2f\n" ${average}` || average=`printf "%'.0f\n" ${average}`
average_disp="${idsCL[LightYellow]} ${average}'watts"
else
average_disp=""
fi
echo -e "${idsCL[White]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_WATT_disp}'watts ~ estimate (${APCPDU_WATT2_disp} watts)${average_disp}${idsCL[Default]}"
echo
if [ "${SVRRM_VOLT}" != "" ]; then
if [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
[[ "${NETRK_WATT}" == *"."* ]] && [[ "${NETRK_WATT}" != *".0"* ]] && NETRK_WATT=`printf "%'.2f\n" ${NETRK_WATT}` || NETRK_WATT=`printf "%'.0f\n" ${NETRK_WATT}`
fi
VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLT}" | bc`
fi
echo -e "${idsCL[White]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}${NETRK_AMP}'amps ~ estimate${idsCL[Default]}"
if [ "${avgdays}" != "" ]; then
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['network-rack']} ${avgdays})
[[ "${average}" == *"."* ]] && [[ "${average}" != *".0"* ]] && average=`printf "%'.2f\n" ${average}` || average=`printf "%'.0f\n" ${average}`
average_disp="${idsCL[LightYellow]} ${average}'watts"
else
average_disp=""
fi
echo -e "${idsCL[White]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}${NETRK_WATT}'watts ~ estimate${average_disp}${idsCL[Default]}"
echo
echo -e "${idsCL[White]}Voltage ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLT}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}"
if [ "${avgdays}" != "" ]; then
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total']} ${avgdays})
[[ "${average}" == *"."* ]] && [[ "${average}" != *".0"* ]] && average=`printf "%'.2f\n" ${average}` || average=`printf "%'.0f\n" ${average}`
average_disp="${idsCL[LightYellow]} ${average}'watts"
else
average_disp=""
fi
echo -e "${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_WATT_disp}'watts${average_disp}${idsCL[Default]}"
echo
fi
if [ "${PTYPE}" == "" ] || [ "${PTYPE}" == "all" ] || [ "${PTYPE}" == "offsite" ]; then
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Offsite Power Information ${avgdayshdr}"
DIVIDER . yellow 75
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
(( $(bc <<<"${OFFSITE_AMPS} < 1") )) && [[ "${OFFSITE_AMPS}" = *"."* ]] && OFFSITE_AMPS="0${OFFSITE_AMPS}"
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
# vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
# OFFSITE_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //')
# OFFSITE_VOLTIN=`echo "scale=2; ${OFFSITE_VOLTIN}/10" | bc`
# OFFSITE_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.1.0 | sed 's/.*: //')
# OFFSITE_VOLTOUT=`echo "scale=2; ${OFFSITE_VOLTOUT}/10" | bc`
# OFFSITE_LOAD=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.3.0 | sed 's/.*: //')
# OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
# OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*900" | bc`
# OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
# OFFSITE_BATT_RUNTIME=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.3.0 | sed 's/.*: //')
# OFFSITE_BATT_CAP=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.4.0 | sed 's/.*: //')
fi
echo -e "${idsCL[White]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTIN}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTOUT}'volts${idsCL[Default]}"
echo
echo -e "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_AMPS}'amps ~ estimate${idsCL[Default]}"
if [ "${avgdays}" != "" ]; then
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['offsite-ups']} ${avgdays})
[[ "${average}" == *"."* ]] && [[ "${average}" != *".0"* ]] && average=`printf "%'.2f\n" ${average}` || average=`printf "%'.0f\n" ${average}`
average_disp="${idsCL[LightYellow]} ${average}'watts"
else
average_disp=""
fi
echo -e "${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_WATTS}'watts ~ estimate${average_disp}${idsCL[Default]}"
echo
echo -e "${idsCL[White]}Battery Capacity ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_CAP}%${idsCL[Default]}"
echo -e "${idsCL[White]}Battery Runtime ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME%.*}'Mins${idsCL[Default]}"
echo
# echo -e "${idsCL[LightCyan]}Current Fan Speed ${idsCL[Default]}: ${idsCL[Green]}${FanSpeed}%${idsCL[Default]}"
# echo
fi
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}ESXi Host Power Information ${avgdayshdr}"
DIVIDER . yellow 75
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
gochk=0
if [ "${PTYPE}" == "" ] || [ "${PTYPE}" == "hosts" ] || ([ "${PTYPE}" == "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${PTYPE}" == "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then
gochk=1
fi
if [ ${gochk} -eq 1 ]; then
# HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.3 | sed 's/.*: //')
# if [ "${HOST_WATTS}" = "" ]; then
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
s=1
for ss in "${HOST_WATT_SENSORS[@]}"; do
[[ "${ss}" = *"System Board"* ]] && break || ((s++))
done
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
# fi
[[ "${HOST_WATTS}" = *"Timeout"* ]] && HOST_WATTS_disp="${idsCL[LightRed]}Host Offline" || HOST_WATTS_disp="${idsCL[Green]}${HOST_WATTS}'watts"
if [ "${avgdays}" != "" ]; then
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays})
[[ "${average}" == *"."* ]] && [[ "${average}" != *".0"* ]] && average=`printf "%'.2f\n" ${average}` || average=`printf "%'.0f\n" ${average}`
average_disp="${idsCL[LightYellow]} ${average}'watts"
else
average_disp=""
fi
echo -e "${idsCL[White]}${SENSOR} ${idsCL[Default]}: ${HOST_WATTS_disp}${average_disp}${idsCL[Default]}"
fi
fi
done
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
[ "${action}" != "" ] && echo
}
CHECKPOWER_SERVICE(){
voltstatus=0
battstatus=0
while true; do
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then
if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then
touch ${PW_TMPFOLDER}/power.ups.low
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
fi
elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
rm -f ${PW_TMPFOLDER}/power.ups.low
fi
if [ "${INVSTATSINFO}" != "" ]; then
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
if [ ${voltstatus} -eq 3 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${INPUTACV}'volts"
battstatus=0
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
if [ ${voltstatus} -eq 0 ]; then
SENDNOTICE "Power Nominal" "Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
fi
fi
voltstatus=1
rm -f ${PW_TMPFOLDER}/power.ac.*
else
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
if [ ${voltstatus} -lt 3 ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power off or low voltage detected" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power off or low voltage detected" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "Power off or low voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts" 1
last_battvolt=${BATTVOLT}
voltstatus=3
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge 180 ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGFILE}
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
SHUTDOWN_SERVER 10.10.2.10 &
touch ${PW_TMPFOLDER}/power.ac.conserve
fi
fi
if (( $(bc <<<"${BATTVOLT} < ${min_battvolt} && ${BATTVOLT} > ${min_battvolt_sys}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 2 && ${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
battstatus=2
fi
elif (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if (( $(bc <<<"${battstatus} < 3 && ${INPUTACV} < ${min_acvolt}") )); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all servers" 1
SHUTDOWN_SYS POWER &
battstatus=3
fi
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 3600 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
if [ ${battstatus} -eq 0 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
fi
battstatus=1
fi
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && temp_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} > 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
else
sleep 20s
fi
######offsite check
# if [ "$(CHECK_HOST 10.2.1.1)" != "false" ]; then
# upsinfo=$(ssh admin@10.2.1.1 'upsc CP1500PFCLCD')
# OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
# OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
# OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
# OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
#
# if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then
# if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
# SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected
# VOLTAGE: ${OFFSITE_VOLTIN}'volts"
# rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
# fi
# if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
# touch ${PW_TMPFOLDER}/power.offsite.restored
#
# elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -ge 60 ]; then
# SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back on after power was restored"
# for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
# HOSTPOWER ${offsite_host} on >/dev/null 2>&1
# done
# rm -f ${PW_TMPFOLDER}/power.offsite.*
# fi
#
# else
# [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Normal Voltage" >> ${PW_LOGPOWER}
# fi
# else
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
#
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 240 ]; then
# SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
# BATT CAPACITY: ${OFFSITE_BATT_CAP}'%25
# BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins" 1
# touch ${PW_TMPFOLDER}/power.offsite.ac.low
# fi
#
# if (( $(bc <<<"${OFFSITE_BATT_CAP} <= 95") )) && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
# SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
# SHUTDOWN_OFFSITE &
# touch ${PW_TMPFOLDER}/power.offsite.off
# fi
# fi
#
#
# fi
sleep 10s
done # &
}
POWERLOGGER_SERVICE(){
while true; do
LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['inv-in']}','${LogDataTime}','${INPUTV}','${INPUTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['inv-out']}','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['battery']}','${LogDataTime}','${BATTV}','${BATTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total']}','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
if [ "${SVRRM_VOLT}" != "" ]; then
APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['server-rack']}','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
else
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
fi
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['network-rack']}','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['offsite-ups']}','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
if [ "${HOST_WATT_SENSORS}" != "" ]; then
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
s=1
for ss in "${HOST_WATT_SENSORS[@]}"; do
[[ "${ss}" = *"System Board"* ]] && break || ((s++))
done
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID[${SENSOR}]}','${LogDataTime}','${HOST_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
fi
done
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then
sleep 1m
else
sleep 5m
fi
done
}
RESTARTALL_SERVICES(){
STOPALL_SERVICES
sleep 10s
STARTALL_SERVICES
}
STOPALL_SERVICES(){
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} stop
done
}
STARTALL_SERVICES(){
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} start
done
}
SERVICE_MNG(){
if [ "${2}" = "start" ]; then
sa=Starting
elif [ "${2}" = "stop" ]; then
sa=Stopping
elif [ "${2}" = "restart" ]; then
sa=Restarting
elif [ "${2}" = "status" ]; then
sa='Status for'
fi
echo -en "${idsCL[LightCyan]}${sa} ${PW_POWERWALL_SERVICES[${1}]} Service ... "
/bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]}
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
}
STOP_SERVICE(){
echo "stop: ${1}"
ps -ef | grep "powerwall.*service ${1} start" | grep -v "grep" | awk '{print $2}' | xargs kill
}
CHECK_SERVICES(){
echo
echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
DIVIDER . yellow 75
for srvc in "${PW_POWERWALL_SERVICES[@]}"; do
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
done
echo
}
SHUTDOWN_MAIN(){
if [ "${1}" == "ServerRoomTH"] || [ "${1}" == "TEMP"]; then
SHUTDOWN_SERVERS MAIN ${1}
reason='Server Room Overheated!'
sendnotice=true
elif [ "${1}" == "POWER"]; then
SHUTDOWN_SERVERS MAIN ${1}
reason='Low Power!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown
${reason}" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
fi
echo
}
SHUTDOWN_SYS(){
if [ "${1}" == "ServerRoomTH"] || [ "${1}" == "TEMP"]; then
SHUTDOWN_SERVERS SYS ${1}
reason='Server Room Overheated!'
sendnotice=true
elif [ "${1}" == "POWER"]; then
SHUTDOWN_SERVERS SYS ${1}
reason='Low Power!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown
${reason}" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
# curl --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd
echo
}
SHUTDOWN_CRIT(){
if [ "${1}" == "Powerwall-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "Powerwall System shutting down" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
sleep 1s
/sbin/poweroff
elif [ "${1}" == "OctoPI-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "OctoPI System shutting down" 1
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
sleep 1s
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
fi
}
SHUTDOWN_OFFSITE(){
if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1
for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
SHUTDOWN_SERVER ${offsite_host} &
done
touch ${PW_FOLDER}/shutdown.offsite
fi
}
SHUTDOWN_SERVERS(){
# TEST=true
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE}
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
if [ "${1^^}" == "MAIN" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.main ]; then
touch ${PW_FOLDER}/shutdown.main
echo -en "Disabling CLS in vCenter ... "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
fi
echo "Done"
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo "Done"
done
# sleep 1m
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
done
fi
elif [ "${1^^}" == "SYS" ]; then
[ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then
touch ${PW_FOLDER}/shutdown.sys
if [ "${1^^}" == "SYS" ]; then
echo -en "Disabling CLS in vCenter ... "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
fi
echo "Done"
fi
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo "Done"
done
# sleep 1m
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
done
fi
elif [ "${1^^}" == "ALL" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.all ]; then
touch ${PW_FOLDER}/shutdown.all
echo -en "Disabling CLS in vCenter ... "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
fi
echo "Done"
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo "Done"
done
# sleep 1m
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
fi
echo "Done"
done
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
done
fi
elif [ "${1^^}" == "OFFSITE" ]; then
if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
touch ${PW_FOLDER}/shutdown.offsite
echo -en "Disabling CLS in vCenter ... "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
fi
echo "Done"
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
echo -en "Stopping VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo "Done"
done
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
done
fi
fi
}
MAINT_MODE_VERIFY_SHUTDOWN(){
ESXIHOST=${1}
MaintMode=""
until [ "${MaintMode}" == "Enabled" ]; do
sleep 3s
MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')
done
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
"$(CHECK_HOST ${ESXIHOST})" == "false" ] && snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2
fi
}
SHUTDOWN_SERVER(){
# TEST=true
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
ESXIHOST=${1}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
tmp=${PW_HOST_IDRACS[${1}]}
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
else
ESXIHOST=''
fi
echo -e "\n${idsCL[Yellow]}ESXi Host Powerdown"
DIVIDER . yellow 75
if [ "${ESXIHOST}" != "" ]; then
host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]}
c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
c=0; spcB=''; spc1=`expr 18 - ${#ESXIHOST} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${ESXIHOST}${spcB}"
echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${PW_HOST_IDRACS[${host}]}${idsCL[Default]}"
DIVIDER . yellow 75
echo
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} (${ESXIHOST})" >> ${PW_LOGFILE}
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
for host in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}] ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
done
echo -e "${idsCL[LightYellow]}All iSCSI-PRI VM's have been sent shutdown signals\n"
sleep 10s
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... "
for host in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}] ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
done
echo -e "${idsCL[LightYellow]}All iSCSI-SYS VM's have been sent shutdown signals\n"
sleep 10s
fi
echo -en "${idsCL[LightCyan]}Enabling maintenance mode & starting iSCSI VM migrations ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
[ "${TEST}" != "true" ] && sleep 25s
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Shutting down VM's ... "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then
[ "${ESXIHOST}" == "10.10.2.10" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
[ "${ESXIHOST}" == "10.10.2.17" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... "
[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
echo -en "${idsCL[LightCyan]}Waiting for Host ${host} (${ESXIHOST}) to shutdown ... "
[ "${TEST}" != "true" ] && while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Powering off SAS Enclosure ... "
[ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
sleep 3s
echo -e "${idsCL[Green]}Done\n"
else
echo -e "${idsCL[LightCyan]}The host ${host} (${ESXIHOST}) will be shutdown momentarily ... \n"
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo
}
ISCSIVMSHUTDOWN(){
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-PRI drive ... "
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
echo
done
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
echo
}
ISCSISYSVMSHUTDOWN(){
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-SYS drive ... "
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
echo
done
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
echo
}
HOSTPOWER(){
echo -e "\n${idsCL[Yellow]}Host Power Control"
DIVIDER . yellow 75
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
idracip=${1}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
tmp=${PW_ESXI_HOST_NAMES[${1}]}
idracip=${PW_HOST_IDRACS[${tmp}]}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[${1}]}
else
idracip=''
fi
if [ "${idracip}" != "" ]; then
host=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
c=0; spcB=''; spc1=`expr 18 - ${#PW_HOST_ADDRESSES[${host}]} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${PW_HOST_ADDRESSES[${host}]}${spcB}"
echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
DIVIDER . yellow 75
echo
if [ "${2,,}" == "on" ] || [ "${2,,}" == "off" ] || [ "${2,,}" == "status" ]; then
if [ "${idracip}" == "10.10.2.20" ] && [ "${2,,}" == "on" ]; then
echo -en "${idsCL[LightCyan]}Powering ON the SAS Enclosure ... "
[ "${TEST}" != "true" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done\n"
fi
echo -e "${idsCL[Green]}"
ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,}
echo -e "${idsCL[Default]}"
if [ "${idracip}" == "10.10.2.20" ] && [ "${2,,}" == "off" ]; then
echo -en "${idsCL[LightCyan]}Powering OFF the SAS Enclosure ... "
[ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done\n"
fi
else
echo -e "${idsCL[LightRed]}'${2,,}' command is not recognized${idsCL[Default]}"
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo -e "\n"
}
LOGROTATE(){
STOPALL_SERVICES
lm=`expr $(date +'%m') - 1`
if [ ${lm} = 0 ]; then
lm=12
ly=`expr $(date +'%Y') - 1`
else
ly=$(date +'%Y')
fi
[ ${lm} -lt 10 ] && lm="0${lm}"
ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
mkdir -p ${ARCHIVEFOLDER}
sleep 5s
mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null
mv ${PW_LOGFOLDER}/service.* ${ARCHIVEFOLDER}/ 2> /dev/null
# source ${PW_FOLDER}/defaults.inc
STARTALL_SERVICES
backupstokeep=(`ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort | tail -12`)
for folder in $(ls -d ${PW_LOGARCHIVE}/* | sort -r); do
keep=0
for filetokeep in ${backupstokeep[@]}; do
if [ "${folder}" == "${filetokeep}" ]; then
keep=1; break
fi
done;
[ ${keep} == 0 ] && rm -Rf ${folder}
done
}
GET_SNAPSHOTS(){
echo
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_ADDRESSES[${SENSOR}]}" != "" ]; then
ESXIHOST=${PW_HOST_ADDRESSES[${SENSOR}]}
echo -e "${idsCL[LightGreen]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]} (${ESXIHOST}) "
DIVIDER . lightGreen 70
echo -en "${idsCL[LighGreen]}Looking for snapshots...${idsCL[Default]}"
snapshots=`${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 ${ESXIHOST}`
if [ "${snapshots}" != "" ]; then
echo -en "\033[2K\033[1A"
echo -e "${snapshots}"
# for snapshot in $(echo ${snapshots} | jq -r '.[] | @base64'); do
# vm=`echo ${snapshot} | base64 --decode | jq -r '.VM'`
# name=`echo ${snapshot} | base64 --decode | jq -r '.Name'`
# created=`echo ${snapshot} | base64 --decode | jq -r '.Created'`
# size=`echo ${snapshot} | base64 --decode | jq -r '.SizeGB'`
# echo "$vm - $name - ${c}reated - $size"
# done
else
echo -e "\033[2K\e[0K\r No snapshots found"
fi
DIVIDER true lightGreen 70
fi
done
}
TEST(){
start=`date +%s`
temptotal=0
daysback=${1}
logsfound=0;
while IFS=$'\t' read date sensorid temp hum ;do
# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
# echo "- ${temptotal} + ${temp}"
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
echo "Logs found: ${logsfound}"
echo "Average: ${average}'F"
echo
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('$(date +'%Y-%m-%d %H:%M:%S')','${INVSTATSINFO}')"
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
echo
}
#########################################################
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
echo
DIVIDER . lightGreen 70
msg="| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})"
c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
DIVIDER . lightGreen 70
fi
# if [ ${action-x} ]; then
case ${action} in
services)
[ "${2}" == "stop" ] && STOPALL_SERVICES
[ "${2}" == "start" ] && STARTALL_SERVICES
[ "${2}" == "restart" ] && RESTARTALL_SERVICES
[ "${2}" == "status" ] && CHECK_SERVICES
;;
dailytemp)
start=`date +%s`
DAILYTEMP
# SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)"
end=`date +%s`; runtime=$((end-start))
echo -e "\nruntime: ${runtime}\n"
;;
checktemp) CHECKTEMP ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
checkpower) CHECKPOWER ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
check)
if [ "${2}" = "power" ]; then
CHECKPOWER ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10}
elif [ "${2}" = "temp" ]; then
CHECKTEMP ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10}
fi
;;
test) CHECKTEMP ${2} ${3};;
iscsi-vm-shutdown)
[ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN
;;
shutdownhost) SHUTDOWN_SERVER ${2};;
shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};;
hostpower) HOSTPOWER ${2} ${3};;
get-snapshots) GET_SNAPSHOTS;;
service)
if [ "${2}" = "all" ]; then
if [ "${3}" == "stop" ]; then
STOPALL_SERVICES
elif [ "${3}" == "start" ]; then
STARTALL_SERVICES
elif [ "${3}" == "restart" ]; then
RESTARTALL_SERVICES
elif [ "${3}" == "status" ]; then
CHECK_SERVICES
fi
else
if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
if [ "${2}" = "temp" ]; then
# SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" &
CHECKTEMP_SERVICE
elif [ "${2}" = "power" ]; then
CHECKPOWER_SERVICE
elif [ "${2}" = "powerlogger" ]; then
POWERLOGGER_SERVICE
fi
elif [ "${3}" = "stop" ]; then
if [ "${2}" = "temp" ]; then
STOP_SERVICE temp
for SENSOR in ${!PW_SENSOR_ID[@]}; do
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
done
elif [ "${2}" = "power" ]; then
STOP_SERVICE power
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER}
elif [ "${2}" = "powerlogger" ]; then
STOP_SERVICE powerlogger
fi
elif [ "${3}" = "restart" ]; then
SERVICE_MNG ${2} stop
slep 2s
SERVICE_MNG ${2} start
elif [ "${3}" = "status" ]; then
srvc=${PW_POWERWALL_SERVICES[${2}]}
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
echo -e "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
echo
fi
fi
;;
logrotate) LOGROTATE;;
update)
# echo -en "${idsCL[LightCyan]}Will reboot monitoring services in the background to apply updates ... "
# RESTARTALL_SERVICES >/dev/null 2>&1 &
# echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
;;
status)
CHECKPOWER
CHECKTEMP
CHECK_SERVICES
;;
average) AVERAGETEMP ${2} ${3} ${4};;
*)
echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}powerwall {option}"
echo
echo -e " ${idsCL[Yellow]}status ${idsCL[Default]}--> PowerWall Status Information"
echo
echo -e " ${idsCL[Yellow]}check {type} {opt} ${idsCL[Default]}--> Check Sensors"
echo -e " ${idsCL[LightYellow]} type = { power / temp }"
echo -e " ${idsCL[LightYellow]} opt.{power} = { all / mainsite / offsite } ${idsCL[Default]}--> Defaults to All"
echo -e " ${idsCL[LightYellow]} opt.{temp} = { avg for # of days to show } ${idsCL[Default]}--> Defaults to not showing averages"
echo
echo -e " ${idsCL[Yellow]}shutdownhost {host} ${idsCL[Default]}--> Shutdown ESXI host and its VMs"
echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
echo
echo -e " ${idsCL[Yellow]}iscsi-vm-shutdown {type} ${idsCL[Default]}--> Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)"
echo -e " ${idsCL[LightYellow]} type = { sys / main }"
echo -e " ${idsCL[LightYellow]} sys='iSCSI-SYS' / main='iSCSI-MAIN' ${idsCL[Default]}--> Defaults to main (iSCSI-PRI)"
echo
echo -e " ${idsCL[Yellow]}shutdown_servers {type} ${idsCL[Default]}--> Shutdown selection of servers"
echo -e " ${idsCL[LightYellow]} type = { main / sys / all }"
echo
echo -e " ${idsCL[Yellow]}hostpower {host} ${idsCL[Default]}--> Control host chassis power"
echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
echo
echo -e " ${idsCL[Yellow]}service {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service"
echo -e " ${idsCL[LightYellow]} srvc = { all / temp / power / powerlogger }"
echo -e " ${idsCL[LightYellow]} cmd = { start / stop / restart / status }"
echo
echo -e " ${idsCL[Yellow]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
echo
;;
esac
# fi
exit 0
Reference in New Issue View Git Blame Copy Permalink