Files
powerwall/inc/power.inc.sh
2023-12-29 13:57:12 -06:00

837 lines
43 KiB
Bash

AVERAGEPOWER(){
watttotal=0
sensorid=${1}
daysback=${2}
logsfound=0;
while IFS=$'\t' read date watt ;do
watttotal=$(bc <<< "scale=2; ${watttotal}+${watt}")
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,watt from power_data WHERE sensorid=${sensorid} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
[ "${logsfound}" != "0" ] && average=$(bc <<< "scale=2; ${watttotal}/${logsfound}") || average=0
# echo "$watttotal-$logsfound=${average}"
# echo "${average}-(${logsfound})"
echo ${average}
}
GET_OFFSITEHOST_FANSPEED(){
export FanSpeed=0
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
export FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
}
CHECKPOWER(){
start=`date +%s`
rm -f ${PW_TMPFOLDER}/*.hostpower*
while [ $# -gt 0 ]; do
case "$1" in
-a|-avg) avgdays=${2};;
-o|-offsite) STYPE=offsite;;
-m|-mainsite) STYPE=mainsite;;
-e|-esxi) PTYPE=esxi;;
-p|-power) PTYPE=power;;
esac
shift 1
done
[ "${avgdays}" == "0" ] && unset avgdays
[ "${avgdays}" != "" ] && avgdayshdr="${avgdays}-Day-Avg" || avgdayshdr=""
[ "${PTYPE}" != "power" ] && GET_OFFSITEHOST_FANSPEED
if ([ "${STYPE}" == "" ] || [ "${STYPE}" == "mainsite" ]) && [ "${PTYPE}" != "esxi" ]; then
echo
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Mainsite Power Information ${idsCL[LightYellow]}${avgdayshdr}"
DIVIDER . yellow 75
echo -en "${idsCL[Yellow]}Pulling data from 'Powerwall' ... "
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
[ "${TEST_INPUTACV}" != "" ] && INPUTV=${TEST_INPUTACV} || INPUTV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
INV_ALARM=`echo ${INVSTATSINFO} | jq '.errors .Alms'`
[ "${INV_ALARM}" == "0" ] && INV_ALARM_disp="${idsCL[Green]}No Alarms (${INV_ALARM})" || INV_ALARM_disp="${idsCL[LightRed]}${PW_INVERTER_ALARMS[${INV_ALARM}]}"
INV_ERROR=`echo ${INVSTATSINFO} | jq '.errors .Err'`
[ "${INV_ERROR}" == "0" ] && INV_ERROR_disp="${idsCL[Green]}No Errors (${INV_ERROR})" || INV_ERROR_disp="${idsCL[LightRed]}${PW_INVERTER_ERRORS[${INV_ERROR}]}"
INV_STATE=`echo ${INVSTATSINFO} | jq '.stats .invSTATES'`
[ "${INV_STATE}" != "0" ] && INV_STATE_disp="${idsCL[Green]}${PW_INVERTER_STATES[${INV_STATE}]}" || INV_STATE_disp="${idsCL[LightRed]}${PW_INVERTER_STATES[${INV_STATE}]}"
INV_MODE=`echo ${INVSTATSINFO} | jq '.stats .power'`
([ "${INV_MODE}" != "3" ] && [ "${INV_MODE}" != "4" ] && [ "${INV_MODE}" != "E" ]) && INV_MODE_disp="${idsCL[Green]}${PW_INVERTER_MODES[${INV_MODE}]}" || INV_MODE_disp="${idsCL[LightRed]}${PW_INVERTER_MODES[${INV_MODE}]}"
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
[ "${TEST_BATTVOLT}" != "" ] && BATTV=${TEST_BATTVOLT} || BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc`
if (( $(bc <<<"${BATTV} >= ${FLOAT_CHARGE}") )); then
if (( $(bc <<<"${BATTV} >= ${BULK_CHARGE}") )); then
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Bulk Charging"
else
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Float Charging"
fi
elif [ "${BATTA}" == "-0.1" ] || [ "${BATTA}" == "-0.2" ] || [ "${BATTA}" == "-0.3" ]; then
BATTA_disp="${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Standby"
else
BATTA_disp="${idsCL[LightRed]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Discharging"
fi
fi
echo -en "\r\033[K"
echo -e "${idsCL[White]}Inverter State ${idsCL[Default]}: ${INV_STATE_disp}"
echo -e "${idsCL[White]}Inverter Mode ${idsCL[Default]}: ${INV_MODE_disp}"
echo -e "${idsCL[White]}Inverter Alarms ${idsCL[Default]}: ${INV_ALARM_disp}"
echo -e "${idsCL[White]}Inverter Errors ${idsCL[Default]}: ${INV_ERROR_disp}"
echo
echo -en "${idsCL[White]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTV}`'volts"
[ "${TEST_INPUTACV}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo ""
echo -e "${idsCL[White]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTA}`'amps"
echo
echo -e "${idsCL[White]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTV}`'volts / `IDS_NUMBER_FORMAT ${OUTPUTV2}`'volts"
echo -e "${idsCL[White]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTA}`'amps / `IDS_NUMBER_FORMAT ${OUTPUTA2}`'amps"
echo -e "${idsCL[White]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTW}`'watts"
echo
echo -en "${idsCL[White]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTV}`'volts ~ `IDS_NUMBER_FORMAT ${BATTLIFE}`%"
[ "${TEST_BATTVOLT}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo ""
echo -e "${idsCL[White]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
# echo
DIVIDER . lightCyan 75
echo -en "${idsCL[Yellow]}Pulling data from 'ServerRoom' ... "
# if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
SVRRM_VOLTIN=`echo "scale=2; ${SVRRM_VOLTIN}/10" | bc`
SVRRM_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLTOUT=`echo "scale=2; ${SVRRM_VOLTOUT}/10" | bc`
SVRRM_WATT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
# fi
#
# if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc`
APCPDU_WATT2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLTOUT}" | bc`
# fi
if [ "${SVRRM_VOLTOUT}" != "" ]; then
if [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
NETRK_WATT2=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT2}" | bc`
fi
VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLTIN}" | bc`
fi
echo -en "\r\033[K"
echo -e "${idsCL[White]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_AMP}`'amps${idsCL[Default]}"
MSG="${idsCL[White]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_WATT}`'watts ${idsCL[Default]}~ estimate"
echo -en "${MSG}"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['server-rack']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
else
echo -en " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${APCPDU_WATT2}` watts)"
fi
echo -e "${idsCL[Default]}"
DIVIDER . lightCyan 75
echo -e "${idsCL[White]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_AMP}`'amps ${idsCL[Default]}~ estimate${idsCL[Green]}${idsCL[Default]}"
MSG="${idsCL[White]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_WATT} 2 true`'watts ${idsCL[Default]}~ estimate"
echo -en "${MSG}"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['network-rack']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
else
echo -en " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${NETRK_WATT2}` watts)"
fi
echo -e "${idsCL[Default]}"
# echo
DIVIDER . lightCyan 75
echo -e "${idsCL[White]}Server Room Voltage IN ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTIN}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}Server Room Voltage OUT ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTOUT}'volts${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}Server Room Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}"
MSG="${idsCL[White]}Server Room Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${SVRRM_WATT}`'watts"
echo -en "${MSG}"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
echo -en "\r\033[K"; echo -e "${MSG}${idsCL[LightYellow]}${average_disp}${idsCL[Default]}"
else
echo -e "${idsCL[Default]}"
fi
echo
fi
if ([ "${STYPE}" == "" ] || [ "${STYPE}" == "offsite" ]) && [ "${PTYPE}" != "esxi" ]; then
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Offsite Power Information ${idsCL[LightYellow]}${avgdayshdr}"
DIVIDER . yellow 75
echo -en "${idsCL[Yellow]}Pulling data from 'Offsite' ... "
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
(( $(bc <<<"${OFFSITE_AMPS} < 1") )) && [[ "${OFFSITE_AMPS}" = *"."* ]] && OFFSITE_AMPS="0${OFFSITE_AMPS}"
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
# vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
# OFFSITE_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //')
# OFFSITE_VOLTIN=`echo "scale=2; ${OFFSITE_VOLTIN}/10" | bc`
# OFFSITE_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.1.0 | sed 's/.*: //')
# OFFSITE_VOLTOUT=`echo "scale=2; ${OFFSITE_VOLTOUT}/10" | bc`
# OFFSITE_LOAD=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.4.2.3.0 | sed 's/.*: //')
# OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
# OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*900" | bc`
# OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
# OFFSITE_BATT_RUNTIME=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.3.0 | sed 's/.*: //')
# OFFSITE_BATT_CAP=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.2.1.33.1.2.4.0 | sed 's/.*: //')
fi
echo -en "\r\033[K"
echo -e "${idsCL[White]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTIN}`'volts${idsCL[Default]}"
echo -e "${idsCL[White]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTOUT}`'volts${idsCL[Default]}"
echo
echo -e "${idsCL[White]}Battery Capacity ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_BATT_CAP}`%${idsCL[Default]}"
echo -e "${idsCL[White]}Battery Runtime ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME}'Mins${idsCL[Default]}"
echo
echo -e "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_AMPS}`'amps ${idsCL[Default]}~ estimate${idsCL[Green]}${idsCL[Default]}"
MSG="${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_WATTS}`'watts ${idsCL[Default]}~ estimate"
echo -en "${MSG}"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['offsite-ups']} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
echo -en "\r\033[K"; echo -e "${MSG}${idsCL[LightYellow]}${average_disp}"
fi
echo -e "${idsCL[Default]}\n"
# echo -e "${idsCL[LightCyan]}Current Fan Speed ${idsCL[Default]}: ${idsCL[Green]}${FanSpeed}'%${idsCL[Default]}"
# echo
fi
if [ "${PTYPE}" != "power" ]; then
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}ESXi Host Power Information ${avgdayshdr}"
DIVIDER . yellow 75
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
gochk=0
if [ "${STYPE}" == "" ] || ([ "${STYPE}" == "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${STYPE}" == "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then
gochk=1
fi
if [ ${gochk} -eq 1 ]; then
echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq 2> /dev/null | sed -e 's/"//g')
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
PS1='0.0'; PS2='0.0'; HOST_WATTS=0
if [ "${HOST_WATT_SENSORS}" != "" ]; then
found=0
for s in {1..5}; do
DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')
if [[ "${DP}" = *"System Board"* ]]; then
found=1
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS1"* ]]; then
PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS2"* ]]; then
PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
fi
done
if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ]; then
[ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}"
[ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}"
HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc`
HOST_WATTS_disp="${idsCL[Green]}${HOST_AMPS}'amps - `IDS_NUMBER_FORMAT ${HOST_WATTS}`'watts"
elif [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep off)" != "" ]; then
HOST_WATTS_disp="${idsCL[LightRed]}Host powered down"
HOST_WATTS=-1
elif [ "${HOST_WATTS}" == "0" ]; then
HOST_WATTS_disp="${idsCL[Yellow]}Host PSU not capable"
else
HOST_WATTS_disp="${idsCL[LightRed]}Could not pull data"
fi
[ "${HOST_WATTS}" == "" ] && HOST_WATTS=0
else
HOST_WATTS_disp="${idsCL[LightRed]}Host offline"
fi
MSG="${idsCL[White]}${SENSOR} ${idsCL[Default]}: ${HOST_WATTS_disp}"
echo -en "\r\033[K${MSG}"
if [ "${HOST_WATT_SENSORS}" != "" ]; then
if [ "${avgdays}" != "" ] && [ ${HOST_WATTS} -gt 0 ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts"
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
elif [ ${HOST_WATTS} -gt 0 ]; then
echo -en "${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]"
fi
if [ "${PW_HOST_IDRACS[${SENSOR}]}" == "10.2.1.21" ] && [ ${HOST_WATTS} -ge 0 ]; then
# vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSOR}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
# FanSpeed=$(ROUND_NUMBER `echo "scale=0; $(ROUND_NUMBER `echo "scale=4; ${vHOSTFAN}/21200" | bc` 2)*100/1" | bc` 0)
until [ "${FanSpeed}" != "0" ] && [ "${FanSpeed}" != "" ]; do
tmp=tmp
done
echo -en "${idsCL[LightCyan]} [FanSpeed=${FanSpeed}'%]"
fi
fi
echo -e "${idsCL[Default]}"
fi
fi
done
fi
end=`date +%s`; runtime=$((end-start)); echo -e "\nruntime: ${runtime}"
[ "${PW_ACTION}" != "" ] && echo
}
CHECKPOWER_SERVICE(){
touch ${PW_TMPFOLDER}/power.ac.start
touch ${PW_TMPFOLDER}/power.dc.start
while true; do
source ${PW_FOLDER}/defaults.inc
[ "${TEST_INPUTACV}" != "" ] || [ "${TEST_BATTVOLT}" != "" ] && TESTMODE=1 || TESTMODE=0
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then
if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then
touch ${PW_TMPFOLDER}/power.ups.low
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
######################### FINISH
fi
elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
rm -f ${PW_TMPFOLDER}/power.ups.low
fi
if [ "${INVSTATSINFO}" != "" ] || [ ${TESTMODE} -eq 1 ]; then
[ "${TEST_INPUTACV}" != "" ] && INPUTACV=${TEST_INPUTACV} || INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
[ "${TEST_BATTVOLT}" != "" ] && BATTVOLT=${TEST_BATTVOLT} || BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
if [ ${TESTMODE} -eq 1 ] && [ ! -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then
touch ${PW_TMPFOLDER}/power.testmode.enabled
SENDNOTICE "TEST MODE" "***TEST MODE*** ENABLED!"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGPOWER}
elif [ ${TESTMODE} -eq 0 ] && [ -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then
rm -f ${PW_TMPFOLDER}/power.testmode.enabled
SENDNOTICE "TEST MODE" "***TEST MODE*** DISABLED!"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGPOWER}
fi
[ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT}
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
[ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT}
if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
VOLTAGE: ${INPUTACV}'volts"
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then
SENDNOTICE "Power Nominal" "Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - 'Check Power' Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.ac.start
fi
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then
if [ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then
SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
touch ${PW_TMPFOLDER}/.power.restored
fi
rm -f ${PW_TMPFOLDER}/power.ac.*
touch ${PW_TMPFOLDER}/power.ac.good
elif [ -f ${PW_TMPFOLDER}/.power.restored ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.power.restored)) -ge $(echo "scale=0; ${PW_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
HOSTMGMT mainsite -p on >/dev/null 2>&1 &
rm -f ${PW_TMPFOLDER}/.power.restored
fi
# UPDATE APC-PDU LINE TO LINE VOLTAGE
CURRENT_VOLTAGE=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //')
SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'`
[ "${SET_VOLTAGE}" != "" ] && [ "${CURRENT_VOLTAGE}" != "" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER}
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.ac.good ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: ${BATTVOLT}'volts" 1
rm -f ${PW_TMPFOLDER}/power.ac.good
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60/1" | bc) ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
SHUTDOWN_SERVER ${PW_MAIN_ISCSI} -esxi off &
touch ${PW_TMPFOLDER}/power.ac.conserve
fi
fi
if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.dc.start
fi
if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then
rm -f ${PW_TMPFOLDER}/power.dc.*
touch ${PW_TMPFOLDER}/power.dc.good
fi
else
[ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_ABNORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow
if [ ! -f ${PW_TMPFOLDER}/shutdown.SYS.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.critlow)) -ge 30 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all remaining servers" 1
SHUTDOWN_SYS POWER &
fi
elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low
[ -f ${PW_TMPFOLDER}/power.dc.critlow ] && rm -f ${PW_TMPFOLDER}/power.dc.critlow
if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.low)) -ge 30 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
fi
elif [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_SHUTDOWN_MAINSERVERS_AFTER}*60" | bc) ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
fi
fi
if [ -f ${PW_TMPFOLDER}/power.ac.low ] || [ -f ${PW_TMPFOLDER}/power.dc.low ]; then
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=2; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} >= 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
fi
else
sleep 20s
fi
######offsite check
# if [ "$(CHECK_HOST 10.2.1.1)" != "false" ]; then
# upsinfo=$(ssh admin@10.2.1.1 'upsc CP1500PFCLCD')
# OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
# OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
# OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
# OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
#
# if (( $(bc <<<"${OFFSITE_VOLTIN} >= ${min_acvolt}") )); then
# if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Back to Normal Voltage" >> ${PW_LOGPOWER}
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
# SENDNOTICE "OFFSITE POWER NOMINAL AGAIN" "Normal voltage detected
# VOLTAGE: ${OFFSITE_VOLTIN}'volts"
# rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
# fi
# if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
# touch ${PW_TMPFOLDER}/power.offsite.restored
#
# elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -ge 60 ]; then
# SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back on after power was restored"
# for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
# HOSTPOWER ${offsite_host} on >/dev/null 2>&1
# done
# rm -f ${PW_TMPFOLDER}/power.offsite.*
# fi
#
# else
# [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -le 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - Offsite Normal Voltage" >> ${PW_LOGPOWER}
# fi
# else
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - OFFSITE POWER OFF/LOW POWER" >> ${PW_LOGFILE}
#
# if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge 240 ]; then
# SENDNOTICE "OFFSITE POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
# BATT CAPACITY: ${OFFSITE_BATT_CAP}'%25
# BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins" 1
# touch ${PW_TMPFOLDER}/power.offsite.ac.low
# fi
#
# if (( $(bc <<<"${OFFSITE_BATT_CAP} <= 95") )) && [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
# SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
# SHUTDOWN_OFFSITE &
# touch ${PW_TMPFOLDER}/shutdown.OFFSITE
# fi
# fi
#
#
# fi
sleep 10s
done # &
}
POWERLOGGER_SERVICE(){
while true; do
LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
source ${PW_FOLDER}/defaults.inc
INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['inv-in']}','${LogDataTime}','${INPUTV}','${INPUTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['inv-out']}','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['battery']}','${LogDataTime}','${BATTV}','${BATTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total']}','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
if [ "${SVRRM_VOLT}" != "" ]; then
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['server-rack']}','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
else
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
fi
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=$(ROUND_NUMBER `echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc` 0)
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['network-rack']}','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['offsite-ups']}','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
if [ "${HOST_WATT_SENSORS}" != "" ]; then
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
for s in {1..5}; do
if [[ "$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')" = *"System Board"* ]]; then
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
if [[ "${HOST_WATTS}" != *"Timeout"* ]] && [ "${HOST_WATTS}" != "0" ]; then
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID[${SENSOR}]}','${LogDataTime}','${HOST_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
break
fi
done
fi
fi
done
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*")" != "" ]; then
sleep ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}m
else
sleep ${PW_MYSQL_LOG_INTERVAL_NORMAL}m
fi
done
}
HOSTPOWER(){
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
idracip=${1}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
tmp=${PW_ESXI_HOST_NAMES[${1}]}
idracip=${PW_HOST_IDRACS[${tmp}]}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
elif ([ "${1::1}" == "o" ] || [ "${1::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${1: -1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[OFF-vMS-Host${1: -1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[${1}]}
else
idracip=''
fi
# if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
# tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
# ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]}
# elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
# ESXI_HOST=${1}
# elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}" != "" ]; then
# ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
# elif ([ "${1::1}" == "o" ] || [ "${1::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${1: -1}]}" != "" ]; then
# ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${1: -1}]}
# elif [ "${PW_HOST_ADDRESSES[${1}]}" != "" ]; then
# ESXI_HOST=${PW_HOST_ADDRESSES[${1}]}
# elif [ "${1^^}" == "ALL" ] || [ "${1^^}" == "MAINSITE" ] || [ "${1^^}" == "OFFSITE" ] || [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "SYS" ]; then
# ESXI_HOST=${HOST^^}
# MN=1
# else
# ESXI_HOST=''
# fi
# [ "${ESXI_HOST}" != "" ] && idracip=${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}]} || idracip=''
echo -e "\n${idsCL[Yellow]}POWERING ${2^^} '${PW_iDRACHOST_SHORTNAMES[${idracip}]}'"
DIVIDER . yellow 75
if [ "${idracip}" != "" ]; then
# host=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
# c=0; spcA=''; spc1=`expr 21 - ${#host} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; c=`expr ${c} + 1`; done
# c=0; spcB=''; spc1=`expr 18 - ${#PW_HOST_ADDRESSES[${host}]} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
# echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${host}${spcA}"
# echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${PW_HOST_ADDRESSES[${host}]}${spcB}"
# echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}"
# DIVIDER . yellow 75
# echo
if [ "${2^^}" == "ON" ] || [ "${2^^}" == "OFF" ] || [ "${2^^}" == "STATUS" ]; then
if [ "${idracip}" == "10.10.2.20" ] && [ "${2^^}" == "ON" ]; then
MSG="Powering ON the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "\n${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
current_status=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 | sed 's/.*: //')
if [ "${current_status}" != "" ] && [ "${current_status}" != "1" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
elif [ "${current_status}" == "1" ]; then
echo -e "${idsCL[Green]}SAS Enclosure Already Powered ON${idsCL[Default]}\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
if [ "${TEST}" != "true" ]; then
current_status=$(ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep "Power is ${2,,}")
if [ "${current_status}" == "" ]; then
echo -e "${idsCL[Green]}"
ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,}
else
echo -e "\n${idsCL[LightCyan]}Host is already Powered ${2^^}${idsCL[Default]}"
fi
else
echo -e "\n${idsCL[Yellow]}(TEST MODE)"
fi
echo -e "${idsCL[Default]}"
if [ "${idracip}" == "10.10.2.20" ] && [ "${2^^}" == "OFF" ]; then
MSG="Powering OFF the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
else
echo -e "${idsCL[LightRed]}'${2^^}' command is not recognized${idsCL[Default]}"
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo -e "${idsCL[Default]}\n"
}
PDUCONTROL(){
function exitspacing {
echo -e "\n\033[K\n\033[K"
exit 0
}
trap exitspacing EXIT
while [ $# -gt 0 ]; do
case "$1" in
-port) PORT_SEL=${2};;
esac
shift 1
done
echo
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}APC-PDU Power Control"
DIVIDER true yellow 75
[ "${PORT_SEL}" != "" ] && PORTSEL=${PORT_SEL}
if [ "${PORTSEL}" == "" ]; then
PORT_DESC=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2 -O vq 2> /dev/null | sed -e 's/"//g')
IFS=$'\n'
read -rd '' -a PORT_DESC <<<"${PORT_DESC}"
unset IFS
p=1
for port in "${PORT_DESC[@]}"; do
portpower=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${p} -O vq 2> /dev/null | sed -e 's/"//g')
[ ${portpower} -eq 1 ] && portpower_disp="${idsCL[Green]}ON" || portpower_disp="${idsCL[LightRed]}OFF"
[ ${p} -lt 10 ] && pp=" ${p}" || pp=${p}
c=0; spc=''; spc1=`expr 28 - ${#port}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
echo -e "${pp}) ${idsCL[LightCyan]}${port}${spc}${idsCL[Default]}: ${portpower_disp}${idsCL[Default]}"
((p++))
done
echo -e ""
echo -e ""
echo -e "(${idsCL[Yellow]}E${idsCL[Default]})xit"
echo -e ""
echo -e "\033[5A"
until [ "${PORTSEL}" != "" ]; do
echo -en "\n${idsCL[LightCyan]}Enter the port number to control: ${idsCL[Default]}"
read PORTSEL
echo
done
fi
[ "${PORTSEL^^}" == "E" ] && exit 0
portsel_power=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g')
[ ${portsel_power} -eq 1 ] && portsel_power_disp="${idsCL[Green]}ON" || portsel_power_disp="${idsCL[LightRed]}OFF"
[ ${portsel_power} -eq 2 ] && portsel_change_power_disp="${idsCL[Green]}ON" || portsel_change_power_disp="${idsCL[LightRed]}OFF"
if [ "${PORT_SEL}" != "" ]; then
portdesc=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2.${PORT_SEL} -O vq 2> /dev/null | sed -e 's/"//g')
echo -e "${idsCL[Yellow]}Port ${PORT_SEL}${idsCL[Default]}) ${idsCL[LightCyan]}${portdesc} ${idsCL[Default]}: ${portsel_power_disp}${idsCL[Default]}\n"
fi
echo -en "${idsCL[LightCyan]}Are you sure you wish to turn ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} (y/N): "
read -n 1 portchange_confirm
case "${portchange_confirm}" in
[Yy])
[ ${portsel_power} -eq 1 ] && portsel_change=2 || portsel_change=1
echo -en "\n\n${idsCL[LightCyan]}Turning ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} ... "
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} i ${portsel_change} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo
if [ "${PORT_SEL}" == "" ]; then
sleep 1s
unset PORTSEL
PDUCONTROL
exit 0
fi
;;
*)
if [ "${PORT_SEL}" == "" ]; then
unset PORTSEL
PDUCONTROL
exit 0
fi
;;
esac
}