2485 lines
115 KiB
Bash
Executable File
2485 lines
115 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
PW_ACTION="${1}"
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/powerwall/defaults.inc
|
|
source ${PW_FOLDER}/settings.conf
|
|
source ${PW_FOLDER}/shutdown-test.sh
|
|
|
|
TEST=true
|
|
|
|
shopt -s lastpipe
|
|
|
|
VERBOSE=false
|
|
|
|
|
|
CHECKTEMP(){
|
|
start=`date +%s`
|
|
cw=25;
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-a|-avg) avgdays=${2};;
|
|
-S|-sensors) temptype="sensors";;
|
|
-e|-esxi) temptype="esxi";;
|
|
-s|-search) search=${2};;
|
|
esac
|
|
shift 1
|
|
done
|
|
echo
|
|
if [ "${avgdays}" != "" ]; then
|
|
[ ${avgdays} -lt 10 ] && avgdays_disp=" ${avgdays}" || avgdays_disp=${avgdays}
|
|
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current / ${idsCL[LightYellow]}${avgdays_disp}DayAVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}"
|
|
else
|
|
avgdays=noavg
|
|
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}"
|
|
fi
|
|
DIVIDER . yellow 90
|
|
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
|
|
if [ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]; then
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ] && [ "${temptype}" != "sensors" ]; then
|
|
vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
|
|
vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}
|
|
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
|
|
echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]${idsCL[Default]}"
|
|
echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
|
|
|
|
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
|
|
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
|
|
IFS=$'\n'
|
|
read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}"
|
|
unset IFS
|
|
echo -en "\r\033[K"
|
|
if [ ${#vHOST_TEMPNAMES[@]} > 1 ]; then
|
|
t=1
|
|
for sensorname in "${vHOST_TEMPNAMES[@]}"; do
|
|
temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
|
|
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
|
|
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
|
|
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
|
|
[ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Calculating ${avgdays} Day average ... "
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays})
|
|
[ "${avgdays}" != "noavg" ] && echo -en "\r\033[K"
|
|
echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}"
|
|
|
|
((t++))
|
|
done
|
|
else
|
|
echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data"
|
|
fi
|
|
|
|
else #host down
|
|
echo -e "\r\033[K${idsCL[LightRed]}${SENSOR} Host is down"
|
|
|
|
# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
|
|
# SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
|
|
# touch ${PW_TMPFOLDER}/${EHOST}.down
|
|
# fi
|
|
fi
|
|
echo
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ] && [ "${temptype}" != "esxi" ]; then
|
|
[[ "${SENSOR}" == *"CPU-Usage"* ]] && echo -en "\033[1A"
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}"
|
|
[ "${avgdays}" != "noavg" ] && echo -en "Pulling data and calculating ${avgdays} Day average ... " || echo -en "Pulling data ... "
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays})
|
|
echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
|
|
echo
|
|
fi
|
|
fi
|
|
done
|
|
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
|
|
[ "${PW_ACTION}" != "" ] && echo
|
|
}
|
|
|
|
CHECKTEMPSENSOR(){
|
|
echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
[ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ] && GAUGESH="'%" || GAUGESH="'F"
|
|
|
|
if [ "${6}" != "noavg" ]; then
|
|
[ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3
|
|
if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then
|
|
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays})
|
|
else
|
|
average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5})
|
|
fi
|
|
else
|
|
average=''
|
|
fi
|
|
|
|
if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
|
|
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
|
|
if [ "${temp_c}" != "" ]; then
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
fi
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
|
|
temp_c=${2}
|
|
temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then
|
|
if [ "${1}" == "Powerwall-CPU-Usage" ]; then
|
|
temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
|
|
else
|
|
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
|
|
fi
|
|
fi
|
|
|
|
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
|
[ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ] && temp_f=$(echo "scale=2; $temp_c/1" | bc -l) || temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
|
|
(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
|
|
[ "${1}" == "ServerRoomTH" ] && temp_f=`echo "scale=1; ${temp_f}+${PW_ServerRoomTH_Dev}" | bc`
|
|
# echo "HERE: '${temp_c}' -> '${temp_f}'"
|
|
# echo -en ""
|
|
c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
|
|
|
|
if [ "${average}" != "" ]; then
|
|
c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
|
|
if [ "${average}" == "" ]; then
|
|
averagedisp="${idsCL[Green]}"
|
|
elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
|
|
averagedisp="${idsCL[Yellow]}${average}"
|
|
elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
|
|
averagedisp="${idsCL[LightRed]}${average}"
|
|
else
|
|
averagedisp="${idsCL[Green]}${average}"
|
|
fi
|
|
fi
|
|
|
|
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
|
|
[ "${average}" != "" ] && echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[Yellow]}WARNING ${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
|
|
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then
|
|
[ "${average}" != "" ] && echo -en "${idsCL[LightRed]}CRITICAL${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[LightRed]}CRITICAL${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
|
|
else
|
|
[ "${average}" != "" ] && echo -en "${idsCL[Green]}Normal ${spct}${temp_f}${GAUGESH} /${spca}${averagedisp}${GAUGESH}${idsCL[Default]}" || echo -en "${idsCL[Green]}Normal ${spct}${temp_f}${GAUGESH}${idsCL[Default]}"
|
|
fi
|
|
|
|
tmp="${spct}${temp_f}"
|
|
c=0; spc=''; spc1=`expr 10 - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
|
|
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
|
|
echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}${GAUGESH}${idsCL[Default]} /${spcc}${idsCL[LightRed]}${temp_crit}${GAUGESH}=>${idsCL[Default]} ]"
|
|
fi
|
|
}
|
|
AVERAGETEMP(){
|
|
temptotal=0
|
|
daysback=${2}
|
|
sensorid=${1}
|
|
logsfound=0;
|
|
if [ "${3}" != "" ]; then
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
|
|
entityid=${entityid[1]}
|
|
WHERE="sensorid=${sensorid} AND entity=${entityid}"
|
|
else
|
|
WHERE="sensorid=${sensorid}"
|
|
fi
|
|
|
|
while IFS=$'\t' read date temp hum ;do
|
|
temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
|
|
((logsfound++))
|
|
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
|
|
average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
|
|
(( $(bc <<<"${average} < 1") )) && [[ "${average}" = *"."* ]] && average="0${average}"
|
|
echo ${average}
|
|
# echo ${logsfound}
|
|
}
|
|
|
|
SENDDAILY(){
|
|
dailtemp=$(DAILYTEMP)
|
|
SENDNOTICE "Daily Temp Readings" "${dailtemp}"
|
|
}
|
|
|
|
CHECKTEMP_SERVICE(){
|
|
cw=20
|
|
declare -A last_temp
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Temp' Service Startup" >> ${PW_LOGFILE}
|
|
while true; do
|
|
# start=`date +%s`
|
|
# time="$(date +%H)$(date +%M)"
|
|
# if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
|
|
# if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
|
|
# touch ${PW_TMPFOLDER}/.sentdaily
|
|
# SENDDAILY &
|
|
# fi
|
|
# fi
|
|
for SENSORa in ${PW_SENSOR_ORD[@]}; do
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
|
|
[ ! -f ${logtemp} ] && touch ${logtemp}
|
|
[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0
|
|
|
|
if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
|
|
# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
|
|
if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
|
|
SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
|
|
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
|
|
fi
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
|
|
IFS=$'\n'
|
|
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
|
|
unset IFS
|
|
[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
|
|
t=1
|
|
|
|
else
|
|
SENSORS_CHECK=(${SENSORa})
|
|
fi
|
|
if [ "${SENSORS_CHECK}" != "" ]; then
|
|
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
|
if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
|
|
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit
|
|
[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
|
|
fi
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
|
echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
|
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
temp_h=''
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
|
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
temp_h=''
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
|
|
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
|
|
temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
|
|
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
|
|
temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
|
|
temp_crit_sys=${temp_crit}
|
|
temp_h=''
|
|
vSENSOR=${SENSOR}
|
|
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
|
((t++))
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
|
if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
|
|
temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
|
|
else
|
|
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
|
|
fi
|
|
temp_h=''
|
|
|
|
fi
|
|
[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
|
|
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
|
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
|
[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGE="CPU-USAGE" || GAUGE="TEMP"
|
|
[ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ] && GAUGESH='%' || GAUGESH="'F"
|
|
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
|
|
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
|
|
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
|
|
|
fi
|
|
|
|
[ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ] && temp_f=$(echo "scale=2; $temp_c/1" | bc -l) || temp_f=$(echo "scale=2; ${temp_c}*1.8 + 32" | bc)
|
|
[ "${SENSOR}" == "ServerRoomTH" ] && temp_f=`echo "scale=2; ${temp_f}+${PW_ServerRoomTH_Dev}" | bc`
|
|
# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
|
|
# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
|
|
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
|
|
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
|
|
fi
|
|
|
|
if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
|
|
if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
|
|
SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
|
|
Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
|
|
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
|
|
|
elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
|
|
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
|
|
# NORMAL TEMP: ${temp_f}${GAUGESH}"
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
|
|
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
|
|
|
elif [ "${relog}" == "1" ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
|
|
|
|
fi
|
|
rm -f ${PW_TMPFOLDER}/temp.*
|
|
rm -f ${PW_TMPFOLDER}/power.${SENSORa}.off
|
|
last_temp[${SENSOR//-/}]=1
|
|
|
|
# iDS-vMS-Offsite Fan Issue Workaround
|
|
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
|
|
vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
|
|
# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
|
|
if [ "${vHOSTFAN}" != "" ]; then
|
|
FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`
|
|
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
|
|
if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
|
|
# if [ ${FanSpeed} -ne 30 ]; then
|
|
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
|
|
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
|
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
else
|
|
([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
|
|
temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
|
|
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
|
|
|
|
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
|
|
touch ${PW_TMPFOLDER}/temp.warn
|
|
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
|
SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
|
last_temp[${SENSOR//-/}]=${temp_f}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
|
|
|
|
# iDS-vMS-Offsite Fan Issue Workaround
|
|
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
|
|
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
|
|
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
|
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
|
|
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
|
|
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
|
SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
|
last_temp[${SENSOR//-/}]=${temp_f}
|
|
|
|
if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
|
|
touch ${PW_TMPFOLDER}/temp.crit
|
|
SHUTDOWN_MAIN ${SENSOR} &
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
|
|
|
|
# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
|
|
# SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
|
|
# SHUTDOWN_SERVER ${SENSORa} &
|
|
# touch ${PW_TMPFOLDER}/power.${SENSORa}.off
|
|
|
|
fi
|
|
fi
|
|
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
|
|
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
|
last_temp[${SENSOR//-/}]=${temp_f}
|
|
if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
|
|
touch ${PW_TMPFOLDER}/temp.critsys
|
|
[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
|
|
SHUTDOWN_CRIT ${SENSOR} &
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
|
|
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
|
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
|
|
else
|
|
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
|
|
fi
|
|
if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
|
if [ "${entityid[1]}" == "" ]; then
|
|
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
|
fi
|
|
entityid=${entityid[1]}
|
|
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
|
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
|
|
${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
|
|
fi
|
|
elif [ "${temp_h}" != "" ]; then
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
|
|
else
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
|
fi
|
|
${mysql_conn} -e "${QRY}"
|
|
touch ${PW_TMPFOLDER}/${SENSOR}.insert
|
|
fi
|
|
else
|
|
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
|
|
SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
|
|
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
|
|
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
|
|
SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
|
|
touch ${PW_TMPFOLDER}/${SENSORa}.down
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
# end=`date +%s`; runtime=$((end-start)); echo "runtime: ${runtime}, pausing for 60secs..."
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
DAILYTEMP(){
|
|
cw=35;
|
|
echo
|
|
echo -e "Sensor Temperature(s) Now / 1DayAVG / 7DayAVG"
|
|
DIVIDER . . 60
|
|
for SENSORa in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
|
|
IFS=$'\n'
|
|
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
|
|
unset IFS
|
|
t=1
|
|
else
|
|
SENSORS_CHECK=(${SENSORa})
|
|
fi
|
|
|
|
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
|
temp_c=$(echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1)
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
|
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
|
|
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
|
|
vSENSOR=${SENSOR}
|
|
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
|
((t++))
|
|
else
|
|
temp_c=''
|
|
|
|
fi
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
|
|
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
|
|
else
|
|
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
|
|
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
|
|
fi
|
|
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
|
temp_f=`echo "scale=2; ${temp_c}*1.8 + 32" | bc`
|
|
echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
|
|
|
|
else
|
|
echo -e "${SENSOR}${spc}No Data"
|
|
|
|
fi
|
|
done
|
|
[ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ] && echo
|
|
done
|
|
}
|
|
|
|
AVERAGEPOWER(){
|
|
watttotal=0
|
|
sensorid=${1}
|
|
daysback=${2}
|
|
logsfound=0;
|
|
while IFS=$'\t' read date watt ;do
|
|
watttotal=$(bc <<< "scale=2; ${watttotal}+${watt}")
|
|
((logsfound++))
|
|
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,watt from power_data WHERE sensorid=${sensorid} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
|
|
[ "${logsfound}" != "0" ] && average=$(bc <<< "scale=2; ${watttotal}/${logsfound}") || average=0
|
|
# echo "$watttotal-$logsfound=${average}"
|
|
# echo "${average}-(${logsfound})"
|
|
echo ${average}
|
|
}
|
|
|
|
CHECKPOWER(){
|
|
start=`date +%s`
|
|
rm -f ${PW_TMPFOLDER}/*.hostpower*
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-a|-avg) avgdays=${2};;
|
|
-o|-offsite) STYPE=offsite;;
|
|
-m|-mainsite) STYPE=mainsite;;
|
|
-e|-esxi) PTYPE=esxi;;
|
|
-p|-power) PTYPE=power;;
|
|
esac
|
|
shift 1
|
|
done
|
|
[ "${avgdays}" = "0" ] && unset avgdays
|
|
[ "${avgdays}" != "" ] && avgdayshdr="${avgdays}-Day-Avg" || avgdayshdr=""
|
|
|
|
if ([ "${STYPE}" == "" ] || [ "${STYPE}" == "mainsite" ]) && [ "${PTYPE}" != "esxi" ]; then
|
|
echo
|
|
DIVIDER . yellow 75
|
|
echo -e "${idsCL[Yellow]}Mainsite Power Information ${idsCL[LightYellow]}${avgdayshdr}"
|
|
DIVIDER . yellow 75
|
|
|
|
echo -en "${idsCL[Yellow]}Pulling data from 'Powerwall' ... "
|
|
|
|
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
|
|
if [ "${INVSTATSINFO}" != "" ]; then
|
|
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
|
|
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
|
|
INV_ALARM=`echo ${INVSTATSINFO} | jq '.errors .Alms'`
|
|
[ "${INV_ALARM}" == "0" ] && INV_ALARM_disp="${idsCL[Green]}No Alarms (${INV_ALARM})" || INV_ALARM_disp="${idsCL[LightRed]}${PW_INVERTER_ALARMS[${INV_ALARM}]}"
|
|
INV_ERROR=`echo ${INVSTATSINFO} | jq '.errors .Err'`
|
|
[ "${INV_ERROR}" == "0" ] && INV_ERROR_disp="${idsCL[Green]}No Errors (${INV_ERROR})" || INV_ERROR_disp="${idsCL[LightRed]}${PW_INVERTER_ERRORS[${INV_ERROR}]}"
|
|
INV_STATE=`echo ${INVSTATSINFO} | jq '.stats .invSTATES'`
|
|
[ "${INV_STATE}" != "0" ] && INV_STATE_disp="${idsCL[Green]}${PW_INVERTER_STATES[${INV_STATE}]}" || INV_STATE_disp="${idsCL[LightRed]}${PW_INVERTER_STATES[${INV_STATE}]}"
|
|
INV_MODE=`echo ${INVSTATSINFO} | jq '.stats .power'`
|
|
([ "${INV_MODE}" != "3" ] && [ "${INV_MODE}" != "4" ] && [ "${INV_MODE}" != "E" ]) && INV_MODE_disp="${idsCL[Green]}${PW_INVERTER_MODES[${INV_MODE}]}" || INV_MODE_disp="${idsCL[LightRed]}${PW_INVERTER_MODES[${INV_MODE}]}"
|
|
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
|
|
OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc`
|
|
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
|
|
OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc`
|
|
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
|
|
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
|
|
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
|
|
BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc`
|
|
if (( $(bc <<<"${BATTV} > ${FLOAT_CHARGE}") )); then
|
|
if (( $(bc <<<"${BATTV} > ${BULK_CHARGE}") )); then
|
|
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Bulk Charging"
|
|
else
|
|
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Float Charging"
|
|
fi
|
|
elif [ "${BATTA}" == "-0.1" ] || [ "${BATTA}" == "-0.2" ] || [ "${BATTA}" == "-0.3" ]; then
|
|
BATTA_disp="${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Standby"
|
|
else
|
|
BATTA_disp="${idsCL[LightRed]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Discharging"
|
|
fi
|
|
fi
|
|
|
|
echo -en "\r\033[K"
|
|
echo -e "${idsCL[White]}Inverter State ${idsCL[Default]}: ${INV_STATE_disp}"
|
|
echo -e "${idsCL[White]}Inverter Mode ${idsCL[Default]}: ${INV_MODE_disp}"
|
|
echo -e "${idsCL[White]}Inverter Alarms ${idsCL[Default]}: ${INV_ALARM_disp}"
|
|
echo -e "${idsCL[White]}Inverter Errors ${idsCL[Default]}: ${INV_ERROR_disp}"
|
|
echo
|
|
echo -e "${idsCL[White]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTV}`'volts"
|
|
echo -e "${idsCL[White]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTA}`'amps"
|
|
echo
|
|
echo -e "${idsCL[White]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTV}`'volts / `IDS_NUMBER_FORMAT ${OUTPUTV2}`'volts"
|
|
echo -e "${idsCL[White]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTA}`'amps / `IDS_NUMBER_FORMAT ${OUTPUTA2}`'amps"
|
|
echo -e "${idsCL[White]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTW}`'watts"
|
|
echo
|
|
echo -e "${idsCL[White]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTV}`'volts ~ `IDS_NUMBER_FORMAT ${BATTLIFE}`%"
|
|
echo -e "${idsCL[White]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
|
|
|
|
# echo
|
|
DIVIDER . lightCyan 75
|
|
|
|
echo -en "${idsCL[Yellow]}Pulling data from 'ServerRoom' ... "
|
|
|
|
# if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
|
|
SVRRM_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
|
|
SVRRM_VOLTIN=`echo "scale=2; ${SVRRM_VOLTIN}/10" | bc`
|
|
SVRRM_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
|
|
SVRRM_VOLTOUT=`echo "scale=2; ${SVRRM_VOLTOUT}/10" | bc`
|
|
|
|
SVRRM_WATT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
|
|
SVRRM_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
|
|
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
|
|
# fi
|
|
#
|
|
# if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
|
|
APCPDU_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
|
|
APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc`
|
|
APCPDU_WATT2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
|
|
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLTOUT}" | bc`
|
|
# fi
|
|
|
|
if [ "${SVRRM_VOLTOUT}" != "" ]; then
|
|
if [ "${APCPDU_AMP}" != "" ]; then
|
|
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
|
|
NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
|
|
NETRK_WATT2=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT2}" | bc`
|
|
fi
|
|
VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLTIN}" | bc`
|
|
fi
|
|
|
|
echo -en "\r\033[K"
|
|
echo -e "${idsCL[White]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_AMP}`'amps${idsCL[Default]}"
|
|
|
|
MSG="${idsCL[White]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_WATT}`'watts ${idsCL[Default]}~ estimate"
|
|
echo -en "${MSG}"
|
|
if [ "${avgdays}" != "" ]; then
|
|
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
|
|
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['server-rack']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
|
|
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
|
|
else
|
|
echo -en " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${APCPDU_WATT2}` watts)"
|
|
fi
|
|
echo -e "${idsCL[Default]}"
|
|
|
|
DIVIDER . lightCyan 75
|
|
|
|
echo -e "${idsCL[White]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_AMP}`'amps ${idsCL[Default]}~ estimate${idsCL[Green]}${idsCL[Default]}"
|
|
MSG="${idsCL[White]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_WATT} 2 true`'watts ${idsCL[Default]}~ estimate"
|
|
echo -en "${MSG}"
|
|
if [ "${avgdays}" != "" ]; then
|
|
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
|
|
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['network-rack']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
|
|
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
|
|
else
|
|
echo -en " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${NETRK_WATT2}` watts)"
|
|
fi
|
|
echo -e "${idsCL[Default]}"
|
|
|
|
# echo
|
|
DIVIDER . lightCyan 75
|
|
|
|
echo -e "${idsCL[White]}Server Room Voltage IN ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTIN}'volts${idsCL[Default]}"
|
|
echo -e "${idsCL[White]}Server Room Voltage OUT ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTOUT}'volts${idsCL[Default]}"
|
|
echo -e "${idsCL[LightCyan]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}"
|
|
echo -e "${idsCL[White]}Server Room Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}"
|
|
|
|
MSG="${idsCL[White]}Server Room Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${SVRRM_WATT}`'watts"
|
|
echo -en "${MSG}"
|
|
if [ "${avgdays}" != "" ]; then
|
|
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
|
|
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
|
|
echo -en "\r\033[K"; echo -e "${MSG}${idsCL[LightYellow]}${average_disp}${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
|
|
fi
|
|
|
|
if ([ "${STYPE}" == "" ] || [ "${STYPE}" == "offsite" ]) && [ "${PTYPE}" != "esxi" ]; then
|
|
|
|
DIVIDER . yellow 75
|
|
echo -e "${idsCL[Yellow]}Offsite Power Information ${idsCL[LightYellow]}${avgdayshdr}"
|
|
DIVIDER . yellow 75
|
|
|
|
echo -en "${idsCL[Yellow]}Pulling data from 'Offsite' ... "
|
|
|
|
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
|
|
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
|
|
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
|
|
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
|
|
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
|
|
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
|
|
(( $(bc <<<"${OFFSITE_AMPS} < 1") )) && [[ "${OFFSITE_AMPS}" = *"."* ]] && OFFSITE_AMPS="0${OFFSITE_AMPS}"
|
|
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
|
|
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
|
|
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
|
|
|
|
# vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
|
|
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
|
|
|
|
# OFFSITE_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //')
|
|
# OFFSITE_VOLTIN=`echo "scale=2; ${OFFSITE_VOLTIN}/10" | bc`
|
|
# OFFSITE_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.1.0 | sed 's/.*: //')
|
|
# OFFSITE_VOLTOUT=`echo "scale=2; ${OFFSITE_VOLTOUT}/10" | bc`
|
|
# OFFSITE_LOAD=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.3.0 | sed 's/.*: //')
|
|
# OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
|
|
# OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*900" | bc`
|
|
# OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
|
|
# OFFSITE_BATT_RUNTIME=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.3.0 | sed 's/.*: //')
|
|
# OFFSITE_BATT_CAP=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.4.0 | sed 's/.*: //')
|
|
|
|
fi
|
|
|
|
echo -en "\r\033[K"
|
|
echo -e "${idsCL[White]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTIN}`'volts${idsCL[Default]}"
|
|
echo -e "${idsCL[White]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTOUT}`'volts${idsCL[Default]}"
|
|
echo
|
|
|
|
echo -e "${idsCL[White]}Battery Capacity ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_BATT_CAP}`%${idsCL[Default]}"
|
|
echo -e "${idsCL[White]}Battery Runtime ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME}'Mins${idsCL[Default]}"
|
|
echo
|
|
|
|
echo -e "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_AMPS}`'amps ${idsCL[Default]}~ estimate${idsCL[Green]}${idsCL[Default]}"
|
|
MSG="${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_WATTS}`'watts ${idsCL[Default]}~ estimate"
|
|
echo -en "${MSG}"
|
|
if [ "${avgdays}" != "" ]; then
|
|
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
|
|
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['offsite-ups']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
|
|
echo -en "\r\033[K"; echo -e "${MSG}${idsCL[LightYellow]}${average_disp}"
|
|
fi
|
|
echo -e "${idsCL[Default]}\n"
|
|
|
|
# echo -e "${idsCL[LightCyan]}Current Fan Speed ${idsCL[Default]}: ${idsCL[Green]}${FanSpeed}'%${idsCL[Default]}"
|
|
# echo
|
|
fi
|
|
|
|
if [ "${PTYPE}" != "power" ]; then
|
|
DIVIDER . yellow 75
|
|
echo -e "${idsCL[Yellow]}ESXi Host Power Information ${avgdayshdr}"
|
|
DIVIDER . yellow 75
|
|
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
|
|
gochk=0
|
|
if [ "${STYPE}" == "" ] || ([ "${STYPE}" == "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${STYPE}" == "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then
|
|
gochk=1
|
|
fi
|
|
if [ ${gochk} -eq 1 ]; then
|
|
echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
|
|
|
|
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq 2> /dev/null | sed -e 's/"//g')
|
|
IFS=$'\n'
|
|
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
|
|
unset IFS
|
|
PS1='0.0'; PS2='0.0'; HOST_WATTS=0
|
|
if [ "${HOST_WATT_SENSORS}" != "" ]; then
|
|
found=0
|
|
for s in {1..5}; do
|
|
DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')
|
|
if [[ "${DP}" = *"System Board"* ]]; then
|
|
found=1
|
|
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
|
|
|
|
elif [[ "${DP}" = *"PS1"* ]]; then
|
|
PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
|
|
elif [[ "${DP}" = *"PS2"* ]]; then
|
|
PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
|
|
fi
|
|
done
|
|
if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ]; then
|
|
[ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}"
|
|
[ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}"
|
|
HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc`
|
|
HOST_WATTS_disp="${idsCL[Green]}${HOST_AMPS}'amps - `IDS_NUMBER_FORMAT ${HOST_WATTS}`'watts"
|
|
|
|
elif [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep off)" != "" ]; then
|
|
HOST_WATTS_disp="${idsCL[LightRed]}Host powered down"
|
|
HOST_WATTS=-1
|
|
|
|
elif [ "${HOST_WATTS}" == "0" ]; then
|
|
HOST_WATTS_disp="${idsCL[Yellow]}Host PSU not capable"
|
|
|
|
else
|
|
HOST_WATTS_disp="${idsCL[LightRed]}Could not pull data"
|
|
|
|
fi
|
|
|
|
[ "${HOST_WATTS}" == "" ] && HOST_WATTS=0
|
|
|
|
else
|
|
HOST_WATTS_disp="${idsCL[LightRed]}Host offline"
|
|
fi
|
|
|
|
MSG="${idsCL[White]}${SENSOR} ${idsCL[Default]}: ${HOST_WATTS_disp}"
|
|
echo -en "\r\033[K${MSG}"
|
|
|
|
if [ "${avgdays}" != "" ] && [ ${HOST_WATTS} -gt 0 ]; then
|
|
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
|
|
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
|
|
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
|
|
elif [ ${HOST_WATTS} -gt 0 ]; then
|
|
echo -en "${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]"
|
|
fi
|
|
if [ "${PW_HOST_IDRACS[${SENSOR}]}" == "10.2.1.21" ] && [ ${HOST_WATTS} -ge 0 ]; then
|
|
vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSOR}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
|
|
FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100/1" | bc`
|
|
# vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSOR}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
|
|
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
|
|
echo -en "${idsCL[LightCyan]} [FanSpeed=${FanSpeed}'%]"
|
|
fi
|
|
|
|
echo -e "${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
|
|
|
|
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
|
|
[ "${PW_ACTION}" != "" ] && echo
|
|
}
|
|
|
|
CHECKPOWER_SERVICE(){
|
|
touch ${PW_TMPFOLDER}/power.ac.start
|
|
touch ${PW_TMPFOLDER}/power.dc.start
|
|
while true; do
|
|
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
|
|
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
|
|
|
|
if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then
|
|
touch ${PW_TMPFOLDER}/power.ups.low
|
|
|
|
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
|
|
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
|
|
|
|
|
|
######################### FINISH
|
|
|
|
|
|
fi
|
|
|
|
elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
|
|
rm -f ${PW_TMPFOLDER}/power.ups.low
|
|
fi
|
|
|
|
if [ "${INVSTATSINFO}" != "" ]; then
|
|
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
|
|
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
|
|
[ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT}
|
|
|
|
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
|
|
[ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT}
|
|
if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
|
|
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}'volts"
|
|
else
|
|
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
|
|
if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then
|
|
SENDNOTICE "Power Nominal" "Service Startup
|
|
Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}'volts
|
|
BATT VOLTAGE: ${BATTVOLT}'volts"
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGFILE}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
|
|
rm -f ${PW_TMPFOLDER}/power.ac.start
|
|
fi
|
|
fi
|
|
if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then
|
|
|
|
if [ -f ${PW_TMPFOLDER}/power.sys.off ] || [ -f ${PW_TMPFOLDER}/power.main.off ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then
|
|
SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
|
|
touch ${PW_TMPFOLDER}/.power.restored
|
|
fi
|
|
|
|
rm -f ${PW_TMPFOLDER}/power.ac.*
|
|
touch ${PW_TMPFOLDER}/power.ac.good
|
|
|
|
elif [ -f ${PW_TMPFOLDER}/.power.restored ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.power.restored)) -ge $(echo "scale=0; ${PW_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
|
|
HOSTMGMT mainsite -p on >/dev/null 2>&1
|
|
rm -f ${PW_TMPFOLDER}/.power.restored
|
|
fi
|
|
|
|
# UPDATE APC-PDU LINE TO LINE VOLTAGE
|
|
CURRENT_VOLTAGE=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //')
|
|
SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'`
|
|
[ "${SET_VOLTAGE}" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER}
|
|
|
|
else
|
|
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER}
|
|
if [ -f ${PW_TMPFOLDER}/power.ac.good ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER}
|
|
SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
|
|
VOLTAGE: ${INPUTACV}'volts
|
|
BATT VOLTAGE: ${BATTVOLT}'volts" 1
|
|
rm -f ${PW_TMPFOLDER}/power.ac.good
|
|
touch ${PW_TMPFOLDER}/power.ac.low
|
|
|
|
##### SHUTDOWN OFFSITE SERVER WHILE ITS AT MAINSITE
|
|
${PW_SCRIPT} host 10.2.1.11 -e off 2>/dev/null &
|
|
|
|
|
|
fi
|
|
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60" | bc) ]; then
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER}
|
|
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
|
|
SHUTDOWN_SERVER ${PW_HOST_IDRACS['iDS-vMS-Host0']} &
|
|
touch ${PW_TMPFOLDER}/power.ac.conserve
|
|
fi
|
|
fi
|
|
|
|
if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then
|
|
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
|
|
if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
|
|
rm -f ${PW_TMPFOLDER}/power.dc.start
|
|
fi
|
|
if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then
|
|
rm -f ${PW_TMPFOLDER}/power.dc.*
|
|
touch ${PW_TMPFOLDER}/power.dc.good
|
|
fi
|
|
|
|
else
|
|
[ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good
|
|
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_ABNORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
|
|
|
|
if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
|
|
[ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow
|
|
|
|
if [ ! -f ${PW_TMPFOLDER}/power.sys.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
|
|
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
|
|
Shutting down all remaining servers" 1
|
|
SHUTDOWN_SYS POWER &
|
|
|
|
fi
|
|
|
|
elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then
|
|
[ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low
|
|
|
|
if [ ! -f ${PW_TMPFOLDER}/power.main.off ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )); then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
|
|
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
|
|
Shutting down main servers" 1
|
|
SHUTDOWN_MAIN POWER &
|
|
|
|
fi
|
|
|
|
elif [ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.low)) -ge $(echo "scale=0; ${PW_SHUTDOWN_MAINSERVERS_AFTER}*60" | bc) ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
|
|
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
|
|
Shutting down main servers" 1
|
|
SHUTDOWN_MAIN POWER &
|
|
fi
|
|
|
|
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
|
|
(( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=1; ${volt_diff} * -1" | bc`
|
|
if (( $(bc <<<"${volt_diff} > 0.5") )); then
|
|
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
|
|
last_battvolt=${BATTVOLT}
|
|
fi
|
|
|
|
fi
|
|
|
|
else
|
|
sleep 20s
|
|
fi
|
|
|
|
######offsite check
|
|
|
|
# if [ "$(CHECK_HOST 10.2.1.1)" != "false" ]; then
|
|
# upsinfo=$(ssh admin@10.2.1.1 'upsc CP1500PFCLCD')
|
|
# OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
|
|
# OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
|
|
# OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
|
|
# OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
|
|
#
|
|
# if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then
|
|
# if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
|
|
# SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected
|
|
# VOLTAGE: ${OFFSITE_VOLTIN}'volts"
|
|
# rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
|
|
# fi
|
|
# if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
|
|
# touch ${PW_TMPFOLDER}/power.offsite.restored
|
|
#
|
|
# elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -ge 60 ]; then
|
|
# SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back on after power was restored"
|
|
# for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
|
|
# HOSTPOWER ${offsite_host} on >/dev/null 2>&1
|
|
# done
|
|
# rm -f ${PW_TMPFOLDER}/power.offsite.*
|
|
# fi
|
|
#
|
|
# else
|
|
# [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Normal Voltage" >> ${PW_LOGPOWER}
|
|
# fi
|
|
# else
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
|
|
#
|
|
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 240 ]; then
|
|
# SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
|
|
# BATT CAPACITY: ${OFFSITE_BATT_CAP}'%25
|
|
# BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins" 1
|
|
# touch ${PW_TMPFOLDER}/power.offsite.ac.low
|
|
# fi
|
|
#
|
|
# if (( $(bc <<<"${OFFSITE_BATT_CAP} <= 95") )) && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
# SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
|
|
# SHUTDOWN_OFFSITE &
|
|
# touch ${PW_TMPFOLDER}/power.offsite.off
|
|
# fi
|
|
# fi
|
|
#
|
|
#
|
|
# fi
|
|
|
|
sleep 10s
|
|
done # &
|
|
}
|
|
|
|
POWERLOGGER_SERVICE(){
|
|
while true; do
|
|
LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
|
|
if [ "${INVSTATSINFO}" != "" ]; then
|
|
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
|
|
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
|
|
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
|
|
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
|
|
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
|
|
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
|
|
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['inv-in']}','${LogDataTime}','${INPUTV}','${INPUTA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['inv-out']}','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['battery']}','${LogDataTime}','${BATTV}','${BATTA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
|
|
if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
|
|
SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
|
|
SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
|
|
SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
|
|
SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
|
|
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total']}','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
|
|
if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
|
|
APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
|
|
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
|
|
if [ "${SVRRM_VOLT}" != "" ]; then
|
|
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['server-rack']}','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
|
|
else
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
|
|
fi
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
|
|
if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then
|
|
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
|
|
NETRK_WATT=`echo "scale=0; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['network-rack']}','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
|
|
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
|
|
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
|
|
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
|
|
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
|
|
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['offsite-ups']}','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
|
|
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
|
|
if [ "${HOST_WATT_SENSORS}" != "" ]; then
|
|
IFS=$'\n'
|
|
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
|
|
unset IFS
|
|
|
|
for s in {1..5}; do
|
|
if [[ "$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')" = *"System Board"* ]]; then
|
|
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
|
|
if [[ "${HOST_WATTS}" != *"Timeout"* ]] && [ "${HOST_WATTS}" != "0" ]; then
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID[${SENSOR}]}','${LogDataTime}','${HOST_WATTS}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
fi
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then
|
|
sleep ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}m
|
|
|
|
else
|
|
sleep ${PW_MYSQL_LOG_INTERVAL_NORMAL}m
|
|
fi
|
|
|
|
done
|
|
}
|
|
|
|
|
|
|
|
RESTARTALL_SERVICES(){
|
|
STOPALL_SERVICES
|
|
sleep 10s
|
|
STARTALL_SERVICES
|
|
}
|
|
STOPALL_SERVICES(){
|
|
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
|
|
SERVICE_MNG ${srvc} stop
|
|
done
|
|
}
|
|
STARTALL_SERVICES(){
|
|
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
|
|
SERVICE_MNG ${srvc} start
|
|
done
|
|
}
|
|
SERVICE_MNG(){
|
|
if [ "${2}" = "start" ]; then
|
|
sa=Starting
|
|
|
|
elif [ "${2}" = "stop" ]; then
|
|
sa=Stopping
|
|
|
|
elif [ "${2}" = "restart" ]; then
|
|
sa=Restarting
|
|
|
|
elif [ "${2}" = "status" ]; then
|
|
sa='Status for'
|
|
|
|
fi
|
|
echo -en "${idsCL[LightCyan]}${sa} ${PW_POWERWALL_SERVICES[${1}]} Service ... "
|
|
/bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]}
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
}
|
|
STOP_SERVICE(){
|
|
echo "stop: ${1}"
|
|
ps -ef | grep "powerwall.*service ${1} start" | grep -v "grep" | awk '{print $2}' | xargs kill
|
|
}
|
|
|
|
CHECK_SERVICES(){
|
|
echo
|
|
echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
|
|
DIVIDER . yellow 75
|
|
|
|
for srvc in "${PW_POWERWALL_SERVICES[@]}"; do
|
|
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
|
|
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
|
|
echo -e "${idsCL[Default]}"
|
|
done
|
|
|
|
echo
|
|
}
|
|
|
|
SHUTDOWN_MAIN(){
|
|
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then
|
|
SHUTDOWN_SERVERS MAIN ${1}
|
|
# HOSTMGMT main -e off >/dev/null 2>&1
|
|
reason='Server Room Overheated!'
|
|
sendnotice=true
|
|
|
|
elif [ "${1}" == "POWER" ]; then
|
|
SHUTDOWN_SERVERS MAIN ${1}
|
|
# HOSTMGMT main -e off >/dev/null 2>&1
|
|
reason='Low Power!'
|
|
sendnotice=true
|
|
|
|
else
|
|
sendnotice=false
|
|
fi
|
|
|
|
if [ "${sendnotice}" == "true" ]; then
|
|
SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown
|
|
${reason}" 1
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
|
|
fi
|
|
|
|
echo
|
|
}
|
|
SHUTDOWN_SYS(){
|
|
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then
|
|
SHUTDOWN_SERVERS SYS ${1}
|
|
# HOSTMGMT sys -e off >/dev/null 2>&1
|
|
reason='Server Room Overheated!'
|
|
sendnotice=true
|
|
|
|
elif [ "${1}" == "POWER" ]; then
|
|
SHUTDOWN_SERVERS SYS ${1}
|
|
# HOSTMGMT sys -e off >/dev/null 2>&1
|
|
reason='Low Power!'
|
|
sendnotice=true
|
|
|
|
else
|
|
sendnotice=false
|
|
fi
|
|
|
|
if [ "${sendnotice}" == "true" ]; then
|
|
SENDNOTICE "SYS SERVERS SHUTDOWN" "SYS servers have been shutdown
|
|
${reason}" 1
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
|
|
fi
|
|
|
|
# curl --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd
|
|
echo
|
|
}
|
|
SHUTDOWN_CRIT(){
|
|
if [ "${1}" == "Powerwall-CPU" ]; then
|
|
SENDNOTICE "${1} TEMP CRITICAL" "Powerwall System shutting down" 1
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
|
|
sleep 1s
|
|
/sbin/poweroff
|
|
|
|
elif [ "${1}" == "OctoPi-CPU" ]; then
|
|
SENDNOTICE "${1} TEMP CRITICAL" "OctoPi System shutting down" 1
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPi System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - OctoPi System shutting down" >> ${PW_LOGFILE}
|
|
sleep 1s
|
|
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
|
|
|
|
elif [ "${1}" == "OctoPi-CPU-Usage" ]; then
|
|
SENDNOTICE "${1} USAGE CRITICAL" "OctoPi System rebooting" 1
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL USAGE - OctoPi System rebooting" >> ${PW_LOGFOLDER}/log-temp-${1}
|
|
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} USAGE CRITICAL - OctoPi System rebooting" >> ${PW_LOGFILE}
|
|
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot
|
|
fi
|
|
}
|
|
SHUTDOWN_OFFSITE(){
|
|
if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1
|
|
for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
|
|
SHUTDOWN_SERVER ${offsite_host} &
|
|
done
|
|
touch ${PW_TMPFOLDER}/power.offsite.off
|
|
fi
|
|
}
|
|
|
|
SHUTDOWN_SERVERS(){
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE}
|
|
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
|
|
|
|
if [ "${1^^}" == "MAIN" ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.main.off ]; then
|
|
touch ${PW_TMPFOLDER}/power.main.off
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Disabling CLS in vCenter"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
MSG="Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
if [ "${2}" != "maintenance" ]; then
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
MSG="Waiting for iSCSI-PRI Host to shutdown"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
MSG="Powering OFF the SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
|
|
fi
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
fi
|
|
|
|
elif [ "${1^^}" == "SYS" ]; then
|
|
[ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP
|
|
if [ ! -f ${PW_TMPFOLDER}/power.sys.off ]; then
|
|
touch ${PW_TMPFOLDER}/power.sys.off
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Disabling CLS in vCenter"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
MSG="Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
if [ "${2}" != "maintenance" ]; then
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
fi
|
|
fi
|
|
|
|
elif [ "${1^^}" == "ALL" ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.all.off ]; then
|
|
touch ${PW_TMPFOLDER}/power.all.off
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Disabling CLS in vCenter"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
|
|
MSG="Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
|
|
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
|
|
MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
|
|
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
|
|
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
if [ "${2}" != "maintenance" ]; then
|
|
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
|
|
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
MSG="Waiting for iSCSI-PRI Host to shutdown"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
MSG="Powering OFF the SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
|
|
fi
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
fi
|
|
|
|
elif [ "${1^^}" == "MAINSITE" ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.main.offsite ]; then
|
|
touch ${PW_TMPFOLDER}/power.main.offsite
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Disabling CLS in vCenter"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Stopping iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
if [ "${2}" != "maintenance" ]; then
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
MSG="Waiting for iSCSI-PRI Host to shutdown"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
MSG="Powering OFF the SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
|
|
fi
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
fi
|
|
|
|
elif [ "${1^^}" == "OFFSITE" ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
touch ${PW_TMPFOLDER}/power.offsite.off
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Disabling CLS in vCenter"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
|
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
fi
|
|
|
|
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
|
|
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
|
|
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
if [ "${2}" != "maintenance" ]; then
|
|
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
|
|
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
fi
|
|
fi
|
|
fi
|
|
}
|
|
MAINT_MODE_VERIFY_SHUTDOWN(){
|
|
ESXIHOST=${1}
|
|
MaintMode=""
|
|
until [ "${MaintMode}" == "Enabled" ]; do
|
|
sleep 3s
|
|
MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')
|
|
done
|
|
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"'
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
|
|
|
|
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
|
|
"$(CHECK_HOST ${ESXIHOST})" == "false" ] && snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2
|
|
fi
|
|
}
|
|
SHUTDOWN_SERVER(){
|
|
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
|
|
tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
|
|
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
|
|
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
|
|
ESXIHOST=${1}
|
|
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
|
|
ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
|
|
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
|
|
tmp=${PW_HOST_IDRACS[${1}]}
|
|
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
|
|
else
|
|
ESXIHOST=''
|
|
fi
|
|
|
|
|
|
if [ "${ESXIHOST}" != "" ]; then
|
|
host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]}
|
|
if [ "${2}" == "maintenance" ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${host} [${ESXIHOST}] into maintenance mode" >> ${PW_LOGFILE}
|
|
else
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} [${ESXIHOST}]" >> ${PW_LOGFILE}
|
|
fi
|
|
|
|
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
|
|
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
|
|
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
|
|
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
|
|
sleep 5s
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
[ "${TEST}" != "true" ] && sleep 10s
|
|
|
|
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
|
|
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... "
|
|
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
|
|
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
|
|
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
|
|
sleep 5s
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
done
|
|
echo
|
|
[ "${TEST}" != "true" ] && sleep 10s
|
|
|
|
fi
|
|
|
|
MSG="Enabling maintenance mode & starting iSCSI VM migrations"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
|
|
sleep 25s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
|
|
MSG="Shutting down remaining VM's"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then
|
|
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
fi
|
|
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
fi
|
|
run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
|
|
if [ "${2}" != "maintenance" ]; then
|
|
MSG="Waiting for VM's to power off or migrate"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
|
|
echo -e "${idsCL[Green]}Continuing in background\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
|
|
|
|
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
|
|
MSG="Waiting for Host ${host} [${ESXIHOST}] to shutdown"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
|
|
MSG="Powering off SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
|
|
else
|
|
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be shutdown momentarily ... \n"
|
|
fi
|
|
else
|
|
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be in maintenance mode momentarily ... \n"
|
|
fi
|
|
|
|
else
|
|
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
}
|
|
|
|
HOSTMGMT(){
|
|
echo -e "\n${idsCL[Yellow]}Host Controls${idsCL[Default]}"
|
|
DIVIDER . yellow 75
|
|
|
|
if [ "${1}" != "-h" ]; then
|
|
HOST=${1}
|
|
if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then
|
|
tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]}
|
|
ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]}
|
|
elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then
|
|
ESXI_HOST=${HOST}
|
|
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${HOST}]}" != "" ]; then
|
|
ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}
|
|
elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then
|
|
ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]}
|
|
elif [ "${1^^}" == "ALL" ] || [ "${1^^}" == "MAINSITE" ] || [ "${1^^}" == "OFFSITE" ] || [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "SYS" ]; then
|
|
ESXI_HOST=${HOST^^}
|
|
hostname="All ${1^^} Host Servers"
|
|
else
|
|
ESXI_HOST=''
|
|
fi
|
|
[ "${ESXI_HOST}" == "" ] && echo -e "Invalid host\n" && exit 1
|
|
if [ "${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}" != "" ]; then
|
|
tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}
|
|
idracip=${PW_HOST_IDRACS[${tmp}]}
|
|
hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
|
|
fi
|
|
fi
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-p|-power) POWER=${2}; MODE="power";;
|
|
-e|-esxi) ESXI=${2}; MODE="esxi";;
|
|
-h)
|
|
echo -e "Usage: ${idsCL[Yellow]}pw host {host} {flags}${idsCL[Default]} {"
|
|
width=35
|
|
printf "%-${width}s- %s\n" " {host}" "(*required, can either be esxiIP,idracIP or Host#)"
|
|
printf "%-${width}s- %s\n" " -p|-power {on / off}" "Controls Host Power"
|
|
printf "%-${width}s- %s\n" " -e|-esxi {off / mm}" "Shuts down ESXi or puts into maintenance mode"
|
|
echo -e "}\n"
|
|
exit 0
|
|
;;
|
|
esac
|
|
shift 1
|
|
done
|
|
|
|
echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${hostname}${spcA}"
|
|
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
|
|
echo -e ""
|
|
else
|
|
c=0; spcA=''; spc1=`expr 21 - ${#hostname} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
|
|
c=0; spcB=''; spc1=`expr 18 - ${#ESXI_HOST} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
|
|
echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${ESXI_HOST}${spcB}"
|
|
echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
|
|
fi
|
|
DIVIDER . yellow 75
|
|
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
|
|
var=PW_${HOST^^}HOSTS[@]; v=1
|
|
for vhost in "${!var}"; do
|
|
echo -e " ${v})${idsCL[LightCyan]} ${PW_ESXI_HOST_NAMES[${vhost}]} [${vhost}]${idsCL[Default]}"
|
|
((v++))
|
|
done
|
|
DIVIDER . yellow 75
|
|
fi
|
|
echo
|
|
|
|
([ "${HOST}" == "" ] || ([ "${POWER}" == "" ] && [ "${ESXI}" == "" ])) && echo -e "Invalid command arguments\n" && exit 1
|
|
|
|
if [ "${MODE}" == "esxi" ]; then
|
|
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
|
|
if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then
|
|
echo -e "${idsCL[LightGreen]}PUTTING '${ESXI_HOST}' HOSTS INTO MAINTENANCE MODE${idsCL[Default]}\n"
|
|
SHUTDOWN_SERVERS ${ESXI_HOST,,} maintenance
|
|
else
|
|
echo -e "${idsCL[LightGreen]}SHUTTING DOWN '${ESXI_HOST}' HOSTS${idsCL[Default]}\n"
|
|
SHUTDOWN_SERVERS ${ESXI_HOST,,}
|
|
fi
|
|
|
|
else
|
|
if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then
|
|
echo -e "${idsCL[LightGreen]}PUTTING HOST INTO MAINTENANCE MODE${idsCL[Default]}\n"
|
|
SHUTDOWN_SERVER ${HOST} maintenance
|
|
else
|
|
echo -e "${idsCL[LightGreen]}SHUTTING DOWN HOST${idsCL[Default]}\n"
|
|
SHUTDOWN_SERVER ${HOST}
|
|
fi
|
|
|
|
fi
|
|
|
|
elif [ "${MODE}" == "power" ]; then
|
|
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
|
|
var=PW_${HOST^^}HOSTS[@]; v=1
|
|
for vhost in "${!var}"; do
|
|
HOSTPOWER ${vhost} ${POWER}
|
|
done
|
|
else
|
|
HOSTPOWER ${HOST} ${POWER}
|
|
fi
|
|
fi
|
|
echo
|
|
}
|
|
|
|
ISCSIVMSHUTDOWN(){
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-PRI drive ... "
|
|
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
|
|
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
|
|
echo
|
|
done
|
|
|
|
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
|
|
echo
|
|
}
|
|
ISCSISYSVMSHUTDOWN(){
|
|
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
|
|
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI-SYS drive ... "
|
|
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
|
|
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
|
|
echo
|
|
done
|
|
|
|
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
|
|
echo
|
|
}
|
|
|
|
HOSTPOWER(){
|
|
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
|
|
idracip=${1}
|
|
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
|
|
tmp=${PW_ESXI_HOST_NAMES[${1}]}
|
|
idracip=${PW_HOST_IDRACS[${tmp}]}
|
|
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
|
|
idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
|
|
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
|
|
idracip=${PW_HOST_IDRACS[${1}]}
|
|
else
|
|
idracip=''
|
|
fi
|
|
|
|
echo -e "\n${idsCL[Yellow]}POWERING ${2^^} '${PW_iDRACHOST_SHORTNAMES[${idracip}]}'"
|
|
DIVIDER . yellow 75
|
|
|
|
if [ "${idracip}" != "" ]; then
|
|
# host=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
|
|
# c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
|
|
# c=0; spcB=''; spc1=`expr 18 - ${#PW_HOST_ADDRESSES[${host}]} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
|
|
# echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
|
|
# echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${PW_HOST_ADDRESSES[${host}]}${spcB}"
|
|
# echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
|
|
# DIVIDER . yellow 75
|
|
# echo
|
|
|
|
if [ "${2^^}" == "ON" ] || [ "${2^^}" == "OFF" ] || [ "${2^^}" == "STATUS" ]; then
|
|
if [ "${idracip}" == "10.10.2.20" ] && [ "${2^^}" == "ON" ]; then
|
|
MSG="Powering ON the SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "\n${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
|
|
if [ "${TEST}" != "true" ]; then
|
|
echo -e "${idsCL[Green]}"
|
|
ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,}
|
|
else
|
|
echo -e "\n${idsCL[Yellow]}(TEST MODE)"
|
|
fi
|
|
echo -e "${idsCL[Default]}"
|
|
|
|
if [ "${idracip}" == "10.10.2.20" ] && [ "${2^^}" == "OFF" ]; then
|
|
MSG="Powering OFF the SAS Enclosure"
|
|
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
|
|
if [ "${TEST}" != "true" ]; then
|
|
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
|
|
sleep 3s
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
|
|
else
|
|
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
|
|
fi
|
|
else
|
|
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
else
|
|
echo -e "${idsCL[LightRed]}'${2^^}' command is not recognized${idsCL[Default]}"
|
|
fi
|
|
else
|
|
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
|
|
fi
|
|
echo -e "${idsCL[Default]}\n"
|
|
}
|
|
|
|
PDUCONTROL(){
|
|
function exitspacing {
|
|
echo -e "\n\033[K\n\033[K"
|
|
exit 0
|
|
}
|
|
trap exitspacing EXIT
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-port) PORT_SEL=${2};;
|
|
esac
|
|
shift 1
|
|
done
|
|
echo
|
|
DIVIDER . yellow 75
|
|
echo -e "${idsCL[Yellow]}APC-PDU Power Control"
|
|
DIVIDER true yellow 75
|
|
[ "${PORT_SEL}" != "" ] && PORTSEL=${PORT_SEL}
|
|
if [ "${PORTSEL}" == "" ]; then
|
|
PORT_DESC=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2 -O vq 2> /dev/null | sed -e 's/"//g')
|
|
IFS=$'\n'
|
|
read -rd '' -a PORT_DESC <<<"${PORT_DESC}"
|
|
unset IFS
|
|
p=1
|
|
for port in "${PORT_DESC[@]}"; do
|
|
portpower=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${p} -O vq 2> /dev/null | sed -e 's/"//g')
|
|
[ ${portpower} -eq 1 ] && portpower_disp="${idsCL[Green]}ON" || portpower_disp="${idsCL[LightRed]}OFF"
|
|
[ ${p} -lt 10 ] && pp=" ${p}" || pp=${p}
|
|
c=0; spc=''; spc1=`expr 28 - ${#port}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
echo -e "${pp}) ${idsCL[LightCyan]}${port}${spc}${idsCL[Default]}: ${portpower_disp}${idsCL[Default]}"
|
|
((p++))
|
|
done
|
|
echo -e ""
|
|
echo -e ""
|
|
echo -e "(${idsCL[Yellow]}E${idsCL[Default]})xit"
|
|
echo -e ""
|
|
echo -e "\033[5A"
|
|
until [ "${PORTSEL}" != "" ]; do
|
|
echo -en "\n${idsCL[LightCyan]}Enter the port number to control: ${idsCL[Default]}"
|
|
read PORTSEL
|
|
echo
|
|
done
|
|
fi
|
|
|
|
[ "${PORTSEL^^}" == "E" ] && exit 0
|
|
|
|
portsel_power=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g')
|
|
[ ${portsel_power} -eq 1 ] && portsel_power_disp="${idsCL[Green]}ON" || portsel_power_disp="${idsCL[LightRed]}OFF"
|
|
[ ${portsel_power} -eq 2 ] && portsel_change_power_disp="${idsCL[Green]}ON" || portsel_change_power_disp="${idsCL[LightRed]}OFF"
|
|
|
|
if [ "${PORT_SEL}" != "" ]; then
|
|
portdesc=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2.${PORT_SEL} -O vq 2> /dev/null | sed -e 's/"//g')
|
|
echo -e "${idsCL[Yellow]}Port ${PORT_SEL}${idsCL[Default]}) ${idsCL[LightCyan]}${portdesc} ${idsCL[Default]}: ${portsel_power_disp}${idsCL[Default]}\n"
|
|
fi
|
|
|
|
echo -en "${idsCL[LightCyan]}Are you sure you wish to turn ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} (y/N): "
|
|
read -n 1 portchange_confirm
|
|
case "${portchange_confirm}" in
|
|
[Yy])
|
|
[ ${portsel_power} -eq 1 ] && portsel_change=2 || portsel_change=1
|
|
echo -en "\n\n${idsCL[LightCyan]}Turning ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} ... "
|
|
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} i ${portsel_change} >/dev/null 2>&1)
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
echo
|
|
if [ "${PORT_SEL}" == "" ]; then
|
|
sleep 1s
|
|
unset PORTSEL
|
|
PDUCONTROL
|
|
exit 0
|
|
fi
|
|
;;
|
|
*)
|
|
if [ "${PORT_SEL}" == "" ]; then
|
|
unset PORTSEL
|
|
PDUCONTROL
|
|
exit 0
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
LOGROTATE(){
|
|
STOPALL_SERVICES
|
|
lm=`expr $(date +'%m') - 1`
|
|
if [ ${lm} = 0 ]; then
|
|
lm=12
|
|
ly=`expr $(date +'%Y') - 1`
|
|
else
|
|
ly=$(date +'%Y')
|
|
fi
|
|
[ ${lm} -lt 10 ] && lm="0${lm}"
|
|
ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
|
|
mkdir -p ${ARCHIVEFOLDER}
|
|
sleep 5s
|
|
mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null
|
|
mv ${PW_LOGFOLDER}/service.* ${ARCHIVEFOLDER}/ 2> /dev/null
|
|
# source ${PW_FOLDER}/defaults.inc
|
|
STARTALL_SERVICES
|
|
backupstokeep=(`ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort | tail -12`)
|
|
for folder in $(ls -d ${PW_LOGARCHIVE}/* | sort -r); do
|
|
keep=0
|
|
for filetokeep in ${backupstokeep[@]}; do
|
|
if [ "${folder}" == "${filetokeep}" ]; then
|
|
keep=1; break
|
|
fi
|
|
done;
|
|
[ ${keep} == 0 ] && rm -Rf ${folder}
|
|
done
|
|
}
|
|
|
|
GET_SNAPSHOTS(){
|
|
echo
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${PW_HOST_ADDRESSES[${SENSOR}]}" != "" ]; then
|
|
ESXIHOST=${PW_HOST_ADDRESSES[${SENSOR}]}
|
|
echo -e "${idsCL[LightGreen]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}] "
|
|
DIVIDER . lightGreen 70
|
|
echo -en "${idsCL[LighGreen]}Looking for snapshots...${idsCL[Default]}"
|
|
snapshots=`${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 ${ESXIHOST}`
|
|
if [ "${snapshots}" != "" ]; then
|
|
echo -en "\033[2K\033[1A"
|
|
echo -e "${snapshots}"
|
|
# for snapshot in $(echo ${snapshots} | jq -r '.[] | @base64'); do
|
|
# vm=`echo ${snapshot} | base64 --decode | jq -r '.VM'`
|
|
# name=`echo ${snapshot} | base64 --decode | jq -r '.Name'`
|
|
# created=`echo ${snapshot} | base64 --decode | jq -r '.Created'`
|
|
# size=`echo ${snapshot} | base64 --decode | jq -r '.SizeGB'`
|
|
# echo "$vm - $name - ${c}reated - $size"
|
|
# done
|
|
|
|
else
|
|
echo -e "\033[2K\e[0K\r No snapshots found"
|
|
fi
|
|
DIVIDER true lightGreen 70
|
|
fi
|
|
done
|
|
}
|
|
|
|
CONFSYNC(){
|
|
rsync -az --timeout=3 ${PW_FOLDER}/settings.conf root@10.2.1.1:${PW_FOLDER}/ & >/dev/null 2>&1
|
|
}
|
|
|
|
TEST(){
|
|
start=`date +%s`
|
|
temptotal=0
|
|
daysback=${1}
|
|
logsfound=0;
|
|
while IFS=$'\t' read date sensorid temp hum ;do
|
|
# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
|
|
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
|
|
# echo "- ${temptotal} + ${temp}"
|
|
((logsfound++))
|
|
|
|
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN
|
|
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
|
|
|
|
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
|
|
|
|
echo "Logs found: ${logsfound}"
|
|
echo "Average: ${average}'F"
|
|
echo
|
|
|
|
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('$(date +'%Y-%m-%d %H:%M:%S')','${INVSTATSINFO}')"
|
|
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "runtime: ${runtime}"
|
|
echo
|
|
}
|
|
|
|
|
|
|
|
#########################################################
|
|
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
|
|
echo
|
|
DIVIDER . lightGreen 70
|
|
msg="| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})"
|
|
c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
|
|
DIVIDER . lightGreen 70
|
|
fi
|
|
# if [ ${PW_ACTION-x} ]; then
|
|
case ${PW_ACTION} in
|
|
services)
|
|
[ "${2}" == "stop" ] && STOPALL_SERVICES
|
|
[ "${2}" == "start" ] && STARTALL_SERVICES
|
|
[ "${2}" == "restart" ] && RESTARTALL_SERVICES
|
|
[ "${2}" == "status" ] && CHECK_SERVICES
|
|
;;
|
|
dailytemp)
|
|
start=`date +%s`
|
|
DAILYTEMP
|
|
# SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)"
|
|
end=`date +%s`; runtime=$((end-start))
|
|
echo -e "\nruntime: ${runtime}\n"
|
|
;;
|
|
checktemp) CHECKTEMP ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
|
|
checkpower) CHECKPOWER ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
|
|
check)
|
|
([ "${2}" == "" ] || ([ "${2}" != "power" ] && [ "${2}" != "temp" ])) && checkrun="-help" || checkrun=${2}
|
|
case ${checkrun} in
|
|
power) CHECKPOWER ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
|
|
temp) CHECKTEMP ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};;
|
|
-h|-help)
|
|
echo -e "\nUsage: ${idsCL[Yellow]}[powerwall or pw] ${idsCL[LightYellow]}check ${idsCL[LightCyan]}{type} {flags} ${idsCL[Default]} {\n"
|
|
width=35
|
|
printf "%-${width}s- %s\n" " {type}" "{power or temp} To check either power or temperatures"
|
|
echo -e "\n ${idsCL[LightCyan]}power${idsCL[Default]} {flag} options"
|
|
printf "%-${width}s- %s\n" " -a|-avg {#days}" "Show average values for X number of days"
|
|
printf "%-${width}s- %s\n" " -m|-mainsite" "Narrow to Mainsite Equipment"
|
|
printf "%-${width}s- %s\n" " -o|-offsite" "Narrow to Offsite Equipment"
|
|
printf "%-${width}s- %s\n" " -p|-power" "Narrow to Power Equipment"
|
|
printf "%-${width}s- %s\n" " -e|-esxi" "Narrow to ESXi Hosts"
|
|
echo -e "\n ${idsCL[LightCyan]}temp${idsCL[Default]} {flag} options"
|
|
printf "%-${width}s- %s\n" " -a|-avg {#days}" "Show average values for X number of days"
|
|
printf "%-${width}s- %s\n" " -S|-sensors" "Narrow to Sensors Only"
|
|
printf "%-${width}s- %s\n" " -e|-esxi" "Narrow to ESXi Hosts Only"
|
|
printf "%-${width}s- %s\n" " -s|-search" "Narrow to search of sensor name"
|
|
echo -e "}\n"
|
|
exit 0
|
|
;;
|
|
esac
|
|
;;
|
|
test) CHECKTEMP ${2} ${3};;
|
|
iscsi-vm-shutdown)
|
|
[ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN
|
|
;;
|
|
shutdownhost) SHUTDOWN_SERVER ${2} ${3};;
|
|
shutdowntest) SHUTDOWN_SERVER2 ${2} ${3};;
|
|
shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};;
|
|
|
|
hostpower) HOSTPOWER ${2} ${3};;
|
|
host) HOSTMGMT ${2} ${3} ${4} ${5} ${6} ${7};;
|
|
|
|
pdu) PDUCONTROL ${2} ${3};;
|
|
|
|
get-snapshots) GET_SNAPSHOTS;;
|
|
|
|
service)
|
|
if [ "${2}" = "all" ]; then
|
|
if [ "${3}" == "stop" ]; then
|
|
STOPALL_SERVICES
|
|
elif [ "${3}" == "start" ]; then
|
|
STARTALL_SERVICES
|
|
elif [ "${3}" == "restart" ]; then
|
|
RESTARTALL_SERVICES
|
|
elif [ "${3}" == "status" ]; then
|
|
CHECK_SERVICES
|
|
fi
|
|
else
|
|
if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
|
|
echo "start: ${1}"
|
|
if [ "${2}" = "temp" ]; then
|
|
# SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" &
|
|
CHECKTEMP_SERVICE
|
|
elif [ "${2}" = "power" ]; then
|
|
CHECKPOWER_SERVICE
|
|
elif [ "${2}" = "powerlogger" ]; then
|
|
POWERLOGGER_SERVICE
|
|
fi
|
|
|
|
elif [ "${3}" = "stop" ]; then
|
|
if [ "${2}" = "temp" ]; then
|
|
STOP_SERVICE temp
|
|
for SENSOR in ${!PW_SENSOR_ID[@]}; do
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
|
|
done
|
|
|
|
elif [ "${2}" = "power" ]; then
|
|
STOP_SERVICE power
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER}
|
|
|
|
elif [ "${2}" = "powerlogger" ]; then
|
|
STOP_SERVICE powerlogger
|
|
|
|
fi
|
|
|
|
elif [ "${3}" = "restart" ]; then
|
|
SERVICE_MNG ${2} stop
|
|
sleep 2s
|
|
SERVICE_MNG ${2} start
|
|
|
|
elif [ "${3}" = "status" ]; then
|
|
srvc=${PW_POWERWALL_SERVICES[${2}]}
|
|
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
echo -e "${idsCL[LightCyan]}${srvc}${spc}: "
|
|
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
|
|
echo -e "${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
fi
|
|
;;
|
|
|
|
logrotate) LOGROTATE;;
|
|
confsync) CONFSYNC;;
|
|
status)
|
|
CHECKPOWER -p ${2} ${3}
|
|
CHECKTEMP -S ${2} ${3}
|
|
CHECK_SERVICES
|
|
;;
|
|
average) AVERAGETEMP ${2} ${3} ${4};;
|
|
oldhelp)
|
|
echo -e "${idsCL[White]} Usage: ${idsCL[LightYellow]}[powerwall or pw]${idsCL[LightCyan]} {option}"
|
|
echo
|
|
echo -e " ${idsCL[LightCyan]}status ${idsCL[Default]}--> PowerWall Status Information"
|
|
echo
|
|
echo -e " ${idsCL[LightCyan]}check${idsCL[Yellow]} {type} {opts} ${idsCL[Default]}--> Check Sensors"
|
|
echo -e " ${idsCL[Yellow]} type = { power / temp }"
|
|
echo -e " ${idsCL[Yellow]} opts ="
|
|
echo -e " ${idsCL[Yellow]} power { all / mainsite / offsite } ${idsCL[Default]}--> Defaults to All"
|
|
echo -e " ${idsCL[Yellow]} temp { avg for # of days to show } ${idsCL[Default]}--> Defaults to not showing averages"
|
|
echo
|
|
echo -e " ${idsCL[LightCyan]}host${idsCL[Yellow]} {host} {flags} ${idsCL[Default]}--> Host Power/ESXi Control"
|
|
echo -e " ${idsCL[Yellow]} host = individual {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
|
|
echo -e " ${idsCL[Yellow]} host = groups { all / mainsite / offsite / main / sys }"
|
|
echo -e " ${idsCL[Yellow]} flags ="
|
|
echo -e " ${idsCL[Yellow]} -p|-power { on / off } ${idsCL[Default]}--> Controls Host Power"
|
|
echo -e " ${idsCL[Yellow]} -e|-esxi { off / mm } ${idsCL[Default]}--> Shuts down ESXi or puts into maintenance mode"
|
|
echo
|
|
# echo -e " ${idsCL[Yellow]}shutdownhost {host} ${idsCL[Default]}--> Shutdown ESXI host and its VMs"
|
|
# echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
|
|
# echo
|
|
# echo -e " ${idsCL[LightCyan]}shutdown_servers${idsCL[Yellow]} {type} ${idsCL[Default]}--> Shutdown selection of servers"
|
|
# echo -e " ${idsCL[Yellow]} type = { main / sys / all }"
|
|
# echo
|
|
echo -e " ${idsCL[LightCyan]}iscsi-vm-shutdown${idsCL[Yellow]} {type} ${idsCL[Default]}--> Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)"
|
|
echo -e " ${idsCL[Yellow]} type = { sys / main }"
|
|
echo -e " ${idsCL[Yellow]} sys='iSCSI-SYS' / main='iSCSI-PRI' ${idsCL[Default]}--> Defaults to main (iSCSI-PRI)"
|
|
echo
|
|
# echo -e " ${idsCL[Yellow]}hostpower {host} {on/off} ${idsCL[Default]}--> Control host chassis power"
|
|
# echo -e " ${idsCL[LightYellow]} host = {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}"
|
|
# echo
|
|
echo -e " ${idsCL[LightCyan]}service${idsCL[Yellow]} {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service"
|
|
echo -e " ${idsCL[Yellow]} srvc = { all / temp / power / powerlogger }"
|
|
echo -e " ${idsCL[Yellow]} cmd = { start / stop / restart / status }"
|
|
echo
|
|
echo -e " ${idsCL[LightCyan]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
|
|
echo
|
|
;;
|
|
*)
|
|
echo -e "\n${idsCL[Yellow]}Commands followed by '-h' will show specific command help. {work-in-progress}${idsCL[Default]}"
|
|
echo
|
|
echo -e "Usage: ${idsCL[LightCyan]}[powerwall or pw] ${idsCL[Yellow]}{COMMAND} ${idsCL[LightYellow]}{FLAGS}${idsCL[Default]} {"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}status${idsCL[Default]} - PowerWall Status Information"
|
|
echo -e " ${idsCL[Yellow]}check${idsCL[Default]} - Check Power or Temperature Sensors"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}host${idsCL[Default]} - Host Power/ESXi Control"
|
|
echo -e " ${idsCL[Yellow]}iscsi-vm-shutdown${idsCL[Default]} - Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}service${idsCL[Default]} - Manage an individual monitor service across multiple nodes"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}update${idsCL[Default]} - Check for and install new PowerWall Script updates"
|
|
echo
|
|
echo -e "}"
|
|
echo
|
|
echo
|
|
exit 0
|
|
;;
|
|
esac
|
|
# fi
|
|
|
|
exit 0 |