voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
3a9cee40fcc6c1c35e3af3f2e5f00a29db609e45
powerwall/powerwall.sh
David Schroeder 3a9cee40fc Update powerwall.sh
2023-11-01 19:19:43 -05:00

1457 lines
62 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power
action="${1}"
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/powerwall/defaults.inc
source ${PW_FOLDER}/settings.conf
# TEST=true
shopt -s lastpipe
VERBOSE=false
CHECKTEMP(){
start=`date +%s`
cw=25;
echo
[ "${1}" != "" ] && [ "${1}" != "noavg" ] && [[ "${1}" =~ ^[0-9]+$ ]] && avgdays=${1} || avgdays=3
# echo -e "${idsCL[Yellow]}Sensor Temperature(s) ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current / ${avgdays}DayAVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[Red]}critical${idsCL[Default]}"
DIVIDER . yellow 90
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${1}" == "" ] || [[ "${SENSOR}" = *"${1}"* ]] || [ "${1}" == "noavg" ] || [[ "${1}" =~ ^[0-9]+$ ]]; then
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ]; then
vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[LightCyan]}- ${vHOSTiDRACIP}${idsCL[Default]}"
[ "${1}" != "noavg" ] && echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..." || echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ..."
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${vHOSTiDRACIP}" == "10.2.1.21" ]; then
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
if [ ${FanSpeed} -gt 90 ]; then
SENDNOTICE "${vHOSTiDRACIP} Fan Issue" "Workaround commands sent to set the Fans at 30%" 1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${vHOSTiDRACIP} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00"
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${vHOSTiDRACIP} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x1E"
fi
fi
IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS
echo -en "\r\033[K"
if [ ${#vHOSTDATA[@]} > 1 ]; then
for sensordata in "${vHOSTDATA[@]}"; do
vSENSOR=$(echo ${sensordata} | cut -d'T' -f1 | xargs)
temp_c=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); temp_c=${temp_c/ /}
temp_warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); temp_warn=${temp_warn/ /}
temp_crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); temp_crit=${temp_crit/ /}
# echo "HERE: ${SENSOR} (${PW_SENSOR_ID[${SENSOR}]}) - ${vSENSOR// /_}"
reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${vSENSOR// /_} ${1})
c=0; spc=''; spc1=`expr ${cw} - ${#vSENSOR} - 1`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: ${reading}"
done
else
echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: No Data"
fi
else #host down
echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down"
# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
# SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
# echo "($datetime) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
# touch ${PW_TMPFOLDER}/${EHOST}.down
# fi
fi
else
echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..."
reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${1})
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
fi
echo
fi
done
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
[ "${action}" != "" ] && echo
}
CHECKTEMPSENSOR(){
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
if [ "${6}" != "noavg" ]; then
[ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3
if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays})
else
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5})
fi
else
average=''
fi
if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
if [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${1}]})" != "false" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
fi
elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
temp_c=${2}
temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`
fi
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
#echo "$temp_c -> $temp_f"
c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ $c = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
if [ "${average}" != "" ]; then
c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
if [ "${average}" == "" ]; then
averagedisp="${idsCL[Green]}"
elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
averagedisp="${idsCL[Yellow]}${average}'F"
elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
averagedisp="${idsCL[Red]}${average}'F"
else
averagedisp="${idsCL[Green]}${average}'F"
fi
else
averagedisp=' '
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
echo -en "${idsCL[Yellow]}WARNING (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Yellow]})${idsCL[Default]}"
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
echo -en "${idsCL[Red]}CRITICAL (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Red]})${idsCL[Default]}"
else
echo -en "${idsCL[Green]}Normal (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Green]})${idsCL[Default]}"
fi
tmp="${spct}${temp_f}"
c=0; spc=''; spc1=`expr 8 - ${#tmp}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F=>${idsCL[Default]} ]"
fi
}
AVERAGETEMP(){
temptotal=0
daysback=${2}
sensorid=${1}
logsfound=0;
if [ "${3}" != "" ]; then
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
entityid=${entityid[1]}
WHERE="sensorid=${sensorid} AND entity=${entityid}"
else
WHERE="sensorid=${sensorid}"
fi
while IFS=$'\t' read date temp hum ;do
temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
echo ${average}
}
SENDDAILY(){
dailtemp=$(DAILYTEMP)
SENDNOTICE "Daily Temp Readings" "${dailtemp}"
}
CHECKTEMP_SERVICE(){
cw=20
declare -A last_temp
while true; do
# start=`date +%s`
if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
touch ${PW_TMPFOLDER}/.sentdaily
SENDDAILY &
fi
fi
for SENSORa in ${PW_SENSOR_ORD[@]}; do
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
[ ! -f ${logtemp} ] && touch ${logtemp}
logtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
[ ${logtime} -ge 3600 ] && relog=1 || relog=0
if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
echo "($datetime) - ${SENSORa} - Sensor is back online" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
fi
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
unset IFS
SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
# SENSORS_CHECK=''
# for i in ${SENSORS_CHECK[@]}; do echo $i; done
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
if [ ${FanSpeed} -gt 92 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30(percent)" 1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00" >/dev/null 2>&1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x1E" >/dev/null 2>&1
FanSpeed=30
fi
fi
else
SENSORS_CHECK=(${SENSORa})
fi
if [ "${SENSORS_CHECK}" != "" ]; then
for SENSOR in "${SENSORS_CHECK[@]}"; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
temp_c=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); temp_c=${temp_c/ /}
temp_warn=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g');
temp_crit=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g');
temp_crit_sys=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g');
temp_warn=`echo $(bc <<< "scale=2; (${temp_warn/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
temp_crit=`echo $(bc <<< "scale=2; (${temp_crit/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
temp_crit_sys=`echo $(bc <<< "scale=2; (${temp_crit_sys/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
temp_h=''
vSENSOR=$(echo ${SENSOR} | cut -d'T' -f1 | xargs)
SENSOR="${SENSORa}-${vSENSOR// /_}"
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "($datetime) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
# echo "HERE: ${SENSOR} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
fi
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80(percent)" 1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00" >/dev/null 2>&1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x50" >/dev/null 2>&1
fi
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
if [ "${TEST}" != "true" ]; then
SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e true -t 0' &
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e false -t 0'
while ping -qw 10 -c3 "${PW_HOST_ADDRESSES[${SENSORa}]}">/dev/null 2>&1; do
sleep 1
done
fi
fi
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
fi
else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
# SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
# NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${PW_TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 30 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30(percent)"
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00" >/dev/null 2>&1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x1E" >/dev/null 2>&1
fi
fi
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
if [ "${entityid[1]}" == "" ]; then
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
fi
entityid=${entityid[1]}
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','${datetime}','${temp_f}')"
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ]; then
QRY2="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','${datetime}','${FanSpeed}')"
${mysql_conn} -e "${QRY2}"
fi
elif [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${PW_TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1
echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
fi
done
fi
elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
echo "($datetime) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSORa}.down
fi
fi
done
# end=`date +%s`; runtime=$((end-start)); echo "runtime: ${runtime}, pausing for 60secs..."
sleep 60
done # &
}
DAILYTEMP(){
cw=35;
echo
echo -e "Sensor Temperature(s) Now / 1DayAVG / 7DayAVG"
DIVIDER . . 60
for SENSORa in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
unset IFS
SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
else
SENSORS_CHECK=(${SENSORa})
fi
for SENSOR in "${SENSORS_CHECK[@]}"; do
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
temp_c=$(echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1)
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
temp_c=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g');
temp_c=${temp_c/ /}
vSENSOR=$(echo ${SENSOR} | cut -d'T' -f1 | xargs)
SENSOR="${SENSORa}-${vSENSOR// /_}"
else
temp_c=''
fi
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
else
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
fi
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
else
echo -e "${SENSOR}${spc}No Data"
fi
done
[ "$temp_c" != "null" ] && [ "$temp_c" != "" ] && echo
done
}
GETHOSTPOWER(){
hostip=${PW_iDRACHOST_SHORTNAMES[${1}]}
hostip=${PW_HOST_ADDRESSES[${hostip}]}
if [ "$(CHECK_HOST ${hostip})" != "false" ]; then
if [ "${1}" == "10.10.2.26" ]; then
temp=0
else
temp=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${1} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Pwr" | awk '/ / {print $6}' | sed -e 's/Watts//g' 2>&1)
fi
else
temp=down
fi
echo ${temp} >| ${PW_TMPFOLDER}/${1}.hostpower
}
check_GETHOSTPOWER(){
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
checked=""
until [ "${checked}" != "" ]; do
[ -f ${PW_TMPFOLDER}/${PW_HOST_IDRACS[${SENSOR}]}.hostpower.done ] && checked=good
done
fi
done
touch ${PW_TMPFOLDER}/done.hostpower
}
CHECKPOWER(){
start=`date +%s`
rm -f ${PW_TMPFOLDER}/*.hostpower*
echo -e "\n${idsCL[Yellow]}Power Information"
DIVIDER . yellow 75
if [ "${1}" == "" ] || [ "${1}" == "hosts" ]; then
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
GETHOSTPOWER ${PW_HOST_IDRACS[${SENSOR}]} &
fi
done
check_GETHOSTPOWER &
fi
if [ "${1}" == "" ] || [ "${1}" == "sensors" ]; then
if [ "$(CHECK_HOST inv1.scity.us)" != "false" ]; then
INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json")
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTV2=`echo "scale=1; ${OUTPUTV}/2" | bc`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTA2=`echo "scale=1; ${OUTPUTA}*2" | bc`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
[[ "${OUTPUTW}" == *"."* ]] && [[ "${OUTPUTW}" != *".0"* ]] && OUTPUTW=`printf "%'.1f\n" ${OUTPUTW}` || OUTPUTW=`printf "%'.0f\n" ${OUTPUTW}`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
[[ "${BATTA}" == *"-"* ]] && BATTA_disp="${idsCL[Green]}${BATTA}'A - Charging" || BATTA_disp="${idsCL[LightRed]}${BATTA}'A - Discharging"
fi
echo -e "${idsCL[LightCyan]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}'V"
echo -e "${idsCL[LightCyan]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}${INPUTA}'A"
echo
echo -e "${idsCL[LightCyan]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}'V / ${OUTPUTV2}'V"
echo -e "${idsCL[LightCyan]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTA}'A / ${OUTPUTA2}'A"
echo -e "${idsCL[LightCyan]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTW}'W"
echo
echo -e "${idsCL[LightCyan]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}${BATTV}'V${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
echo
if [ "$(CHECK_HOST 10.10.1.205)" != "false" ]; then
SVRRM_VOLT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
SVRRM_WATT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
[[ "${SVRRM_WATT}" == *"."* ]] && [[ "${SVRRM_WATT}" != *".0"* ]] && SVRRM_WATT_disp=`printf "%'.1f\n" ${SVRRM_WATT}` || SVRRM_WATT_disp=`printf "%'.0f\n" ${SVRRM_WATT}`
SVRRM_AMP=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
fi
if [ "$(CHECK_HOST 10.10.1.210)" != "false" ]; then
APCPDU_AMP=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
# APCPDU_WATT=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
[[ "${APCPDU_WATT}" == *"."* ]] && [[ "${APCPDU_WATT}" != *".0"* ]] && APCPDU_WATT_disp=`printf "%'.1f\n" ${APCPDU_WATT}` || APCPDU_WATT_disp=`printf "%'.0f\n" ${APCPDU_WATT}`
fi
echo -e "${idsCL[LightCyan]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_AMP}'A${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}${APCPDU_WATT_disp}'W${idsCL[Default]}"
echo
if [ "${SVRRM_VOLT}" != "" ]; then
if [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
[[ "${NETRK_WATT}" == *"."* ]] && [[ "${NETRK_WATT}" != *".0"* ]] && NETRK_WATT=`printf "%'.1f\n" ${NETRK_WATT}` || NETRK_WATT=`printf "%'.0f\n" ${NETRK_WATT}`
fi
VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLT}" | bc`
fi
echo -e "${idsCL[LightCyan]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}${NETRK_AMP}'A${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}${NETRK_WATT}'W${idsCL[Default]}"
echo
echo -e "${idsCL[LightCyan]}Voltage ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLT}'V${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'V${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'A${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_WATT_disp}'W${idsCL[Default]}"
echo
fi
########
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Offsite Power Information"
DIVIDER true yellow 75
if [ "$(CHECK_HOST 10.2.1.100)" != "false" ]; then
OFFSITE_VOLTIN=$(snmpwalk -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //')
OFFSITE_VOLTIN=`echo "scale=1; ${OFFSITE_VOLTIN}/10" | bc`
OFFSITE_VOLTOUT=$(snmpwalk -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.1.0 | sed 's/.*: //')
OFFSITE_VOLTOUT=`echo "scale=1; ${OFFSITE_VOLTOUT}/10" | bc`
OFFSITE_LOAD=$(snmpwalk -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.3.0 | sed 's/.*: //')
OFFSITE_WATTS=`echo "scale=1; (${OFFSITE_LOAD}/100)*900" | bc`
OFFSITE_AMPS=`echo "scale=1; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
fi
echo -e "${idsCL[LightCyan]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTIN}'V${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_VOLTOUT}'V${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Total Amps ~ ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_AMPS}'A${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Total Wattage ~ ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_WATTS}'W${idsCL[Default]}"
echo
########
if [ "${1}" == "" ] || [ "${1}" == "hosts" ]; then
finalcheck=""
until [ "${finalcheck}" != "" ]; do
for hostfile in ${PW_TMPFOLDER}/*.hostpower; do
if [ -f ${hostfile} ] && [ "${hostfile}" != "${PW_TMPFOLDER}/*.hostpower" ]; then
host=${hostfile%.*}; host=${host##*/}
if [ "${host}" != "*" ] && [ "${host}" != "done" ]; then
watts=$(cat ${hostfile})
if [ "${watts}" == "down" ]; then
hostpower="${idsCL[LightRed]}Host Offline"
elif [ "${watts}" == "0" ]; then
hostpower="${idsCL[Default]}n/a"
else
[[ "${watts}" == *"."* ]] && [[ "${watts}" != *".0"* ]] && watts=`printf "%'.1f\n" ${watts}` || watts=`printf "%'.0f\n" ${watts}`
hostpower="${idsCL[Green]}${watts}'W"
fi
echo -e "${idsCL[LightCyan]}HOST '${PW_iDRACHOST_SHORTNAMES[${host}]}' ${idsCL[Default]}: ${hostpower}${idsCL[Default]}"
mv ${hostfile} ${hostfile}.done
elif [ "${host}" == "done" ]; then
finalcheck=done
rm -f ${PW_TMPFOLDER}/*.hostpower*
fi
fi
done
sleep 0.25
done
fi
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
[ "${action}" != "" ] && echo
}
CHECKPOWER_SERVICE(){
voltstatus=0
battstatus=0
# rm -f ${PW_TMPFOLDER}/powerlogger.insert
while true; do
INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
# [ -f ${PW_TMPFOLDER}/powerlogger.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/powerlogger.insert)) || lastinsert=100000
# if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
# wait=60
# else
# wait=300
# fi
# if [ ${lastinsert} -ge ${wait} ]; then
# POWERLOGGER "${INVSTATSINFO}" &
# fi
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
if [ $voltstatus -eq 3 ]; then
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
VOLTAGE: ${INPUTACV}V"
echo -e "($datetime) - ${INPUTACV}V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
battstatus=0
else
errtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${PW_LOGPOWER}
if [ $voltstatus -eq 0 ]; then
SENDNOTICE "Power Nominal" "($datetime) Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V"
echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
echo -e "($datetime) - ${INPUTACV}V / ${BATTVOLT}V - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
fi
fi
voltstatus=1
rm -f ${PW_TMPFOLDER}/power.ac.*
else
echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
if [ $voltstatus -lt 3 ]; then
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "($datetime) Power off or low voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V" 1
last_battvolt=$BATTVOLT
fi
voltstatus=3
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
Shutting down main servers" 1
SHUTDOWN_MAIN power &
battstatus=2
fi
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
touch ${PW_TMPFOLDER}/power.dc.low
if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
Shutting down all servers" 1
SHUTDOWN_SYS power &
battstatus=3
fi
else
errtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${PW_LOGPOWER}
if [ $battstatus -eq 0 ]; then
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
fi
rm -f ${PW_TMPFOLDER}/power.dc.*
battstatus=1
fi
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
if [ $volt_diff -gt 0 ]; then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=$BATTVOLT
fi
else
sleep 20s
fi
if [ "${archivedone}" != "complete" ] && [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0000 ] && [ $(date +%H)$(date +%M) -le 0005 ]; then
LOGROTATE &
archivedone=complete
# elif [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0100 ] && [ "${archivedone}" = "complete" ]; then
# archivedone=false
fi
sleep 10s
done # &
}
POWERLOGGER(){
INVSTATSINFO=${1}
datetime=`date +'%Y-%m-%d %H:%M:%S'`
# INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json")
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
SVRRM_VOLT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.2.1 | sed 's/.*: //')
SVRRM_WATT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.4.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.3.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=1; ${SVRRM_AMP}/10" | bc`
APCPDU_AMP=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
# APCPDU_WATT=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${datetime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${datetime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${datetime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
touch ${PW_TMPFOLDER}/powerlogger.insert
}
POWERLOGGER_SERVICE(){
while true; do
datetime=`date +'%Y-%m-%d %H:%M:%S'`
INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json")
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
SVRRM_VOLT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.2.1 | sed 's/.*: //')
SVRRM_WATT=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.4.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -v 1 -c public 10.10.1.205 iso.3.6.1.2.1.33.1.4.4.1.3.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=1; ${SVRRM_AMP}/10" | bc`
APCPDU_AMP=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
# APCPDU_WATT=$(snmpwalk -v 1 -c public 10.10.1.210 .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT=`echo "scale=1; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
NETRK_AMP=`echo "scale=1; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('11','${datetime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('12','${datetime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('13','${datetime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
sleep 1m
else
sleep 5m
fi
done
}
RESTARTALL_SERVICES(){
STOPALL_SERVICES
sleep 10s
STARTALL_SERVICES
}
STOPALL_SERVICES(){
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} stop
done
}
STARTALL_SERVICES(){
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} start
done
}
SERVICE_MNG(){
if [ "${2}" = "start" ]; then
sa=Starting
elif [ "${2}" = "stop" ]; then
sa=Stopping
elif [ "${2}" = "restart" ]; then
sa=Restarting
elif [ "${2}" = "status" ]; then
sa='Status for'
fi
echo -en "${idsCL[LightCyan]}${sa} ${PW_POWERWALL_SERVICES[${1}]} Service ... "
/bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]}
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
}
STOP_SERVICE(){
echo "stop: ${1}"
ps -ef | grep "powerwall.*service ${1} start" | grep -v "grep" | awk '{print $2}' | xargs kill
}
CHECK_SERVICES(){
echo
echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
DIVIDER . yellow 75
for srvc in "${PW_POWERWALL_SERVICES[@]}"; do
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
done
echo
}
SHUTDOWN_MAIN(){
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS MAIN ${2}
reason='Server Room Overheated!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown
${reason}" 1
echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
fi
echo
}
SHUTDOWN_SYS(){
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS SYS ${2}
reason='Server Room Overheated!'
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown
${reason}" 1
echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
# curl --data "cmd=PWRoff" http://10.10.0.61/cmd
echo
}
SHUTDOWN_CRIT(){
if [ "${1}" == "Powerwall-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) Powerwall System shutting down" 1
echo -e "($datetime) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($datetime) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
sleep 1s
/sbin/poweroff
elif [ "${1}" == "OctoPI-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) OctoPI System shutting down" 1
echo -e "($datetime) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
echo -e "($datetime) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
sleep 1s
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
fi
}
SHUTDOWN_SERVERS(){
# HOSTS=${1}HOSTS[@]
# for ESXIHOST in "${!HOSTS}"; do
# echo "Shutting Down: ${ESXIHOST}"
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# done
echo "($datetime) - Shutting down ${1} servers" >> ${PW_LOGFILE}
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
if [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "ALL" ]; then
if [ ! -f $FOLDER/shutdown.main ]; then
touch $FOLDER/shutdown.main
echo -en "Disabling CLS in vCenter ... "
if [ "${TEST}" != "true" ]; then
ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
fi
echo "Done"
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
echo "Done"
done
sleep 1m
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "Verifying iSCSI VM's stopped on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..."
if [ "${TEST}" != "true" ]; then
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
fi
echo "Done"
done
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
sleep 1
done
fi
echo "Done"
done
fi
fi
if [ "${1^^}" == "SYS" ] || [ "${1^^}" == "ALL" ]; then
[ ! -f $FOLDER/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
if [ ! -f $FOLDER/shutdown.sys ]; then
touch $FOLDER/shutdown.sys
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
echo "Done"
done
sleep 1m
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Suspending MySQL & LAN-DNS VM's on: ${ESXIHOST} ... "
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Shutting Down Host: ${ESXIHOST} ... "
if [ "${TEST}" != "true" ]; then
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
fi
echo "Done"
done
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
sleep 1
done
fi
echo "Done"
done
fi
fi
}
SHUTDOWN_SERVER(){
# TEST=true
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
hostip=${PW_HOST_ADDRESSES[${tmp}]}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
hostip=${1}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
hostip=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
tmp=${PW_HOST_IDRACS[${1}]}
hostip=${PW_HOST_ADDRESSES[${tmp}]}
else
hostip=''
fi
echo -e "\n${idsCL[Yellow]}ESXi Host Powerdown"
DIVIDER . yellow 75
if [ "${hostip}" != "" ]; then
host=${PW_ESXI_HOST_NAMES[${hostip}]}
c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
c=0; spcB=''; spc1=`expr 18 - ${#hostip} - 1`; until [ $c = ${spc1} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${hostip}${spcB}"
echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${PW_HOST_IDRACS[${host}]}${idsCL[Default]}"
DIVIDER . yellow 75
echo
echo "($datetime) - Shutting down ${host} (${hostip})" >> ${PW_LOGFILE}
echo -en "${idsCL[LightCyan]}Enabling maintenance mode & starting iSCSI VM migrations ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${hostip}]} & >/dev/null 2>&1)
sleep 30s
echo -e "${idsCL[Green]}Done\n"
# echo -en "${idsCL[LightCyan]}Giving time for iSCSI VM's to begin migration to other hosts ... "
# [ "${TEST}" != "true" ] && sleep 45s
# echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Shutting down remaining VM's ... "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${hostip} >/dev/null 2>&1)
# ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${hostip} >/dev/null 2>&1
fi
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Verifying no VMs remain on the host ... "
[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown-verify.ps1 ${hostip} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Shutting down Host ... "
[ "${TEST}" != "true" ] && run=$(ssh root@${hostip} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' >/dev/null 2>&1)
echo -e "${idsCL[Green]}Host will begin shutdown in 10secs\n"
echo -en "${idsCL[LightCyan]}Disabling maintenance mode ... "
[ "${TEST}" != "true" ] && run=$(ssh root@${hostip} 'esxcli system maintenanceMode set -e false -t 0' >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done\n"
echo -en "${idsCL[LightCyan]}Waiting for Host ${host} (${hostip}) to shutdown ... "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${hostip}">/dev/null 2>&1; do
sleep 1
done
fi
echo -e "${idsCL[Green]}Done\n"
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo
}
ISCSIVMSHUTDOWN(){
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-PRI drive ... "
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
echo
done
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
echo
}
ISCSISYSVMSHUTDOWN(){
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-SYS drive ... "
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
echo
done
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
echo
}
HOSTPOWER(){
echo -e "\n${idsCL[Yellow]}Host Power Control"
DIVIDER . yellow 75
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
idracip=${1}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
tmp=${PW_ESXI_HOST_NAMES[${1}]}
idracip=${PW_HOST_IDRACS[${tmp}]}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[${1}]}
else
idracip=''
fi
if [ "${idracip}" != "" ]; then
host=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
c=0; spcB=''; spc1=`expr 18 - ${#PW_HOST_ADDRESSES[${host}]} - 1`; until [ $c = ${spc1} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${PW_HOST_ADDRESSES[${host}]}${spcB}"
echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
DIVIDER . yellow 75
echo
if [ "${2,,}" == "on" ] || [ "${2,,}" == "off" ] || [ "${2,,}" == "status" ]; then
ssh root@${PW_RACADM_ACCESS} ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "'${PW_ESXI_PASS}'" chassis power ${2,,}
else
echo -e "${idsCL[LightRed]}'${2,,}' command is not recognized${idsCL[Default]}"
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo -e "\n"
}
LOGROTATE(){
# STOPALL_SERVICES
lm=`expr $(date +'%m') - 1`
if [ $lm = 0 ]; then
lm=12
ly=`expr $(date +'%Y') - 1`
else
ly=$(date +'%Y')
fi
ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
mkdir -p ${ARCHIVEFOLDER}
mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/
# source ${PW_FOLDER}/defaults.inc
# STARTALL_SERVICES
}
GET_SNAPSHOTS(){
echo
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_ADDRESSES[${SENSOR}]}" != "" ]; then
ESXIHOST=${PW_HOST_ADDRESSES[${SENSOR}]}
echo -e "${idsCL[LightGreen]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]} (${ESXIHOST}) "
DIVIDER . lightGreen 70
echo -en "${idsCL[LighGreen]}Looking for snapshots...${idsCL[Default]}"
snapshots=`${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 ${ESXIHOST}`
if [ "${snapshots}" != "" ]; then
echo -en "\033[2K\033[1A"
echo -e "${snapshots}"
# for snapshot in $(echo ${snapshots} | jq -r '.[] | @base64'); do
# vm=`echo ${snapshot} | base64 --decode | jq -r '.VM'`
# name=`echo ${snapshot} | base64 --decode | jq -r '.Name'`
# created=`echo ${snapshot} | base64 --decode | jq -r '.Created'`
# size=`echo ${snapshot} | base64 --decode | jq -r '.SizeGB'`
# echo "$vm - $name - $created - $size"
# done
else
echo -e "\033[2K\e[0K\r No snapshots found"
fi
DIVIDER true lightGreen 70
fi
done
}
TEST(){
start=`date +%s`
temptotal=0
daysback=${1}
logsfound=0;
while IFS=$'\t' read date sensorid temp hum ;do
# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
# echo "- ${temptotal} + ${temp}"
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
echo "Logs found: ${logsfound}"
echo "Average: ${average}'F"
echo
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
echo
}
#########################################################
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
echo
DIVIDER . lightGreen 70
msg="| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})"
c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
DIVIDER . lightGreen 70
fi
# if [ ${action-x} ]; then
case $action in
services)
[ "${2}" == "stop" ] && STOPALL_SERVICES
[ "${2}" == "start" ] && STARTALL_SERVICES
[ "${2}" == "restart" ] && RESTARTALL_SERVICES
[ "${2}" == "status" ] && CHECK_SERVICES
;;
checktemp) CHECKTEMP ${2};;
dailytemp)
start=`date +%s`
DAILYTEMP
# SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)"
end=`date +%s`; runtime=$((end-start))
echo -e "\nruntime: ${runtime}\n"
;;
checkpower) CHECKPOWER ${2};;
test) CHECKTEMP ${2} ${3};;
iscsi-vm-shutdown)
[ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN
;;
shutdownhost) SHUTDOWN_SERVER ${2};;
shutdown_servers) SHUTDOWN_SERVERS ${2};;
hostpower) HOSTPOWER ${2} ${3};;
get-snapshots) GET_SNAPSHOTS;;
service)
if [ "${2}" = "all" ]; then
if [ "${3}" == "stop" ]; then
STOPALL_SERVICES
elif [ "${3}" == "start" ]; then
STARTALL_SERVICES
elif [ "${3}" == "restart" ]; then
RESTARTALL_SERVICES
elif [ "${3}" == "status" ]; then
CHECK_SERVICES
fi
else
if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
if [ "${2}" = "temp" ]; then
SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" &
CHECKTEMP_SERVICE
elif [ "${2}" = "power" ]; then
CHECKPOWER_SERVICE
elif [ "${2}" = "powerlogger" ]; then
POWERLOGGER_SERVICE
fi
elif [ "${3}" = "stop" ]; then
if [ "${2}" = "temp" ]; then
STOP_SERVICE temp
for SENSOR in ${!PW_SENSOR_ID[@]}; do
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
done
elif [ "${2}" = "power" ]; then
STOP_SERVICE power
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER}
elif [ "${2}" = "powerlogger" ]; then
STOP_SERVICE powerlogger
fi
elif [ "${3}" = "restart" ]; then
SERVICE_MNG ${2} stop
slep 2s
SERVICE_MNG ${2} start
elif [ "${3}" = "status" ]; then
srvc=${PW_POWERWALL_SERVICES[${2}]}
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
echo
fi
fi
;;
update);;
status)
CHECKPOWER
CHECKTEMP
CHECK_SERVICES
;;
average) AVERAGETEMP ${2} ${3} ${4};;
*)
echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}powerwall {option}"
echo
echo -e " ${idsCL[Yellow]}status ${idsCL[Default]}--> PowerWall Status Information"
echo -e " ${idsCL[Yellow]}checktemp {searchstring} ${idsCL[Default]}--> Check Temperature Sensors"
echo -e " ${idsCL[Yellow]}checkpower {avg} ${idsCL[Default]}--> Check Power Readings"
echo -e " ${idsCL[LightYellow]} avg = { noavg='Does not display average' / "
echo -e " #='Num of days for avg' }${idsCL[Default]} --> Defaults to 3 days avg"
echo
echo -e " ${idsCL[Yellow]}shutdownhost {host} ${idsCL[Default]}--> Shutdown ESXI host and its VMs"
echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
echo -e " ${idsCL[Yellow]}iscsi-vm-shutdown {type} ${idsCL[Default]}--> Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)"
echo -e " ${idsCL[LightYellow]} type = { sys='iSCSI-SYS' / "
echo -e " main='iSCSI-MAIN' }${idsCL[Default]} --> Defaults to main (iSCSI-PRI)"
echo -e " ${idsCL[Yellow]}shutdown_servers {type} ${idsCL[Default]}--> Shutdown selection of servers"
echo -e " ${idsCL[LightYellow]} type = { main / sys / all }"
echo -e " ${idsCL[Yellow]}hostpower {host} ${idsCL[Default]}--> Control host chassis power"
echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
echo
# echo
# echo -e " ${idsCL[Yellow]}services {cmd} ${idsCL[Default]}--> Manage all monitor services at once"
# echo -e " ${idsCL[LightYellow]} cmd = {start/stop/restart/status}"
# echo
echo -e " ${idsCL[Yellow]}service {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service"
echo -e " ${idsCL[LightYellow]} srvc = { all / temp / power / powerlogger }"
echo -e " ${idsCL[LightYellow]} cmd = { start / stop / restart / status }"
echo
echo -e " ${idsCL[Yellow]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
echo
;;
esac
# fi
exit 0
Reference in New Issue View Git Blame Copy Permalink