Files
powerwall/inc/power.inc.bak.sh
2024-05-02 21:56:36 -05:00

1091 lines
58 KiB
Bash

AVERAGEPOWER(){
sensorid=${1}
daysback=${2}
if [[ "${daysback^^}" = *"M"* ]]; then
interval="${daysback::-1} MINUTE"
elif [[ "${daysback^^}" = *"H"* ]]; then
interval="${daysback::-1} HOUR"
elif [[ "${daysback^^}" = *"D"* ]]; then
interval="${daysback::-1} DAY"
elif [[ "${daysback}" =~ ^[0-9]+$ ]]; then
interval="${daysback} DAY"
fi
if [ "${3}" = "both" ]; then
logsfound_amp=0;
logsfound_watt=0;
total_amp=0
total_watt=0
while IFS=$'\t' read date amp watt ;do
if [ "${amp}" != "" ] && [ "${amp}" != "NULL" ]; then
total_amp=$(bc <<< "scale=2; ${total_amp}+${amp}")
((logsfound_amp++))
fi
if [ "${watt}" != "" ] && [ "${watt}" != "NULL" ]; then
total_watt=$(bc <<< "scale=2; ${total_watt}+${watt}")
((logsfound_watt++))
fi
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,amp,watt from power_data WHERE sensorid=${sensorid} AND date >= DATE_SUB(NOW(), INTERVAL ${interval}) ORDER BY id DESC;")
if [ "${logsfound_watt}" != "0" ]; then
average_amp=$(bc <<< "scale=2; ${total_amp}/${logsfound_amp}")
average_watt=$(bc <<< "scale=2; ${total_watt}/${logsfound_watt}")
else
average_amp=0
average_watt=0
fi
echo "${average_amp}~${average_watt}"
else
logsfound=0;
total=0
[ "${3}" != "" ] && ptype=${3} || ptype=watt
while IFS=$'\t' read date ${ptype} ;do
if [ "${!ptype}" != "" ] && [ "${!ptype}" != "NULL" ]; then
total=$(bc <<< "scale=2; ${total}+${!ptype}")
((logsfound++))
fi
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,${ptype} from power_data WHERE sensorid=${sensorid} AND date >= DATE_SUB(NOW(), INTERVAL ${interval}) ORDER BY id DESC;")
[ "${logsfound}" != "0" ] && average=$(bc <<< "scale=2; ${total}/${logsfound}") || average=0
echo ${average}
fi
}
CHECKPOWER(){
start=`date +%s`
rm -f ${PW_TMPFOLDER}/*.hostpower*
REPORT=0
while [ $# -gt 0 ]; do
case "$1" in
-a|-avg) avgdays=${2};;
-o|-offsite) STYPE=offsite;;
-m|-mainsite) STYPE=mainsite;;
-e|-esxi) PTYPE=esxi;;
-p|-power) PTYPE=power;;
-r|-report)
REPORT=1
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
;;
esac
shift 1
done
if [ "${avgdays}" != "" ]; then
if [ "${avgdays}" = "0" ]; then
unset avgdays
elif [[ "${avgdays^^}" = *"M"* ]]; then
avgdays_disp="${avgdays::-1}Min"
elif [[ "${avgdays^^}" = *"H"* ]]; then
avgdays_disp="${avgdays::-1}Hr"
elif [[ "${avgdays^^}" = *"D"* ]]; then
avgdays_disp="${avgdays::-1}Day"
elif [[ "${avgdays}" =~ ^[0-9]+$ ]]; then
avgdays_disp="${avgdays}Day"
fi
# else
# avgdays=noavg
fi
if ([ "${STYPE}" = "" ] || [ "${STYPE}" = "mainsite" ]) && [ "${PTYPE}" != "esxi" ]; then
[ ${REPORT} -eq 0 ] && echo
[ ${REPORT} -eq 0 ] && DIVIDER . yellow ${PRI_WIDTH}
echo -en "${idsCL[Yellow]}Mainsite Power Information"
[ "${avgdays}" != "" ] && echo -e " ${idsCL[LightYellow]}${avgdays_disp}-AVG" || echo
DIVIDER . yellow ${PRI_WIDTH}
[ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'Powerwall' ... "
INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
[ "${TEST_INPUTACV}" != "" ] && INPUTV=${TEST_INPUTACV} || INPUTV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
INV_ALARM=`echo ${INVSTATSINFO} | jq '.errors .Alms'`
[ "${INV_ALARM}" = "0" ] && INV_ALARM_disp="${idsCL[Green]}No Alarms (${INV_ALARM})" || INV_ALARM_disp="${idsCL[LightRed]}${PW_INVERTER_ALARMS[${INV_ALARM}]}"
INV_ERROR=`echo ${INVSTATSINFO} | jq '.errors .Err'`
[ "${INV_ERROR}" = "0" ] && INV_ERROR_disp="${idsCL[Green]}No Errors (${INV_ERROR})" || INV_ERROR_disp="${idsCL[LightRed]}${PW_INVERTER_ERRORS[${INV_ERROR}]}"
INV_STATE=`echo ${INVSTATSINFO} | jq '.stats .invSTATES'`
[ "${INV_STATE}" != "0" ] && INV_STATE_disp="${idsCL[Green]}${PW_INVERTER_STATES[${INV_STATE}]}" || INV_STATE_disp="${idsCL[LightRed]}${PW_INVERTER_STATES[${INV_STATE}]}"
INV_MODE=`echo ${INVSTATSINFO} | jq '.stats .power'`
([ "${INV_MODE}" != "3" ] && [ "${INV_MODE}" != "4" ] && [ "${INV_MODE}" != "E" ]) && INV_MODE_disp="${idsCL[Green]}${PW_INVERTER_MODES[${INV_MODE}]}" || INV_MODE_disp="${idsCL[LightRed]}${PW_INVERTER_MODES[${INV_MODE}]}"
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
[ "${TEST_BATTVOLT}" != "" ] && BATTV=${TEST_BATTVOLT} || BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc`
if (( $(bc <<<"${BATTV} >= ${FLOAT_CHARGE}") )); then
if (( $(bc <<<"${BATTV} >= ${BULK_CHARGE}") )); then
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Bulk Charging"
else
BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Float Charging"
fi
elif [ "${BATTA}" = "-0.1" ] || [ "${BATTA}" = "-0.2" ] || [ "${BATTA}" = "-0.3" ]; then
BATTA_disp="${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Standby"
else
BATTA_disp="${idsCL[LightRed]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Discharging"
fi
fi
[ ${REPORT} -eq 0 ] && echo -en "\r\033[K"
echo -e "${idsCL[White]}Inverter State ${idsCL[Default]}: ${INV_STATE_disp}"
if [ ${REPORT} -eq 0 ]; then
echo -e "${idsCL[White]}Inverter Mode ${idsCL[Default]}: ${INV_MODE_disp}"
echo -e "${idsCL[White]}Inverter Alarms ${idsCL[Default]}: ${INV_ALARM_disp}"
echo -e "${idsCL[White]}Inverter Errors ${idsCL[Default]}: ${INV_ERROR_disp}"
echo
fi
echo -en "${idsCL[White]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTV}`'volts"
[ "${TEST_INPUTACV}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo ""
echo -e "${idsCL[White]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTA}`'amps"
if [ ${REPORT} -eq 0 ]; then
echo
echo -e "${idsCL[White]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTV}`'volts / `IDS_NUMBER_FORMAT ${OUTPUTV2}`'volts"
echo -e "${idsCL[White]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTA}`'amps / `IDS_NUMBER_FORMAT ${OUTPUTA2}`'amps"
echo -e "${idsCL[White]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTW} 0`'watts"
echo
fi
echo -en "${idsCL[White]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTV}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)"
[ "${TEST_BATTVOLT}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo ""
echo -e "${idsCL[White]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}"
# echo
DIVIDER . lightCyan ${PRI_WIDTH}
[ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'ServerRoom' ... "
# if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
SVRRM_VOLTIN=`echo "scale=2; ${SVRRM_VOLTIN}/10" | bc`
SVRRM_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLTOUT=`echo "scale=2; ${SVRRM_VOLTOUT}/10" | bc`
SVRRM_WATT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
AIRCON_INFO=$(curl -m 3 -s http://${PW_REMOTE_SENSOR['ServerRoom-AC']}:8888/status/${PW_SENSOR_DEVICEID['ServerRoom-AC']} | jq '.dps')
AIRCON_AMP=$(echo "${AIRCON_INFO}" | grep '"18":' | cut -d' ' -f 4 | sed 's/,//g')
AIRCON_AMP=$(ROUND_NUMBER $(echo "scale=3; ${AIRCON_AMP}/1000" | bc -l) 2)
AIRCON_WATT=$(echo "${AIRCON_INFO}" | grep '"19":' | cut -d' ' -f 4 | sed 's/,//g')
AIRCON_WATT=$(echo "scale=1; ${AIRCON_WATT}/10" | bc -l)
SVRRM_AMP_WAC=$(echo "scale=2; ${SVRRM_AMP}+${AIRCON_AMP}" | bc -l)
SVRRM_WATT_WAC=$(echo "scale=2; ${SVRRM_WATT}+${AIRCON_WATT}" | bc -l)
# fi
#
# if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc`
APCPDU_WATT2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //')
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLTOUT}" | bc`
# fi
if [ "${SVRRM_VOLTOUT}" != "" ]; then
if [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc`
NETRK_WATT2=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT2}" | bc`
fi
VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLTIN}" | bc`
fi
if [ ${REPORT} -eq 0 ]; then
echo -en "\r\033[K"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[Yellow]}Calculating ServerRack Averages ... "
APCPDU_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['server-rack']} ${avgdays} both)
APCPDU_AMP_avg=$(echo ${APCPDU_avg} | cut -d'~' -f 1)
APCPDU_WATT_avg=$(echo ${APCPDU_avg} | cut -d'~' -f 2)
echo -en "\r\033[K"
fi
echo -en "${idsCL[White]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_AMP} 2 yes`'amps"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${APCPDU_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[White]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_WATT} 0`'watts ${idsCL[Default]}~ est"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${APCPDU_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${APCPDU_WATT2}` watts)${idsCL[Default]}"
DIVIDER . lightCyan ${PRI_WIDTH}
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[Yellow]}Calculating NetworkRack Averages ... "
NETRK_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['network-rack']} ${avgdays} both)
NETRK_AMP_avg=$(echo ${NETRK_avg} | cut -d'~' -f 1)
NETRK_WATT_avg=$(echo ${NETRK_avg} | cut -d'~' -f 2)
echo -en "\r\033[K"
fi
echo -en "${idsCL[White]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_AMP} 2 yes`'amps ${idsCL[Default]}~ est"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${NETRK_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[White]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_WATT} 0`'watts ${idsCL[Default]}~ est"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${NETRK_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${NETRK_WATT2} 0` watts) ${idsCL[Default]}~ est${idsCL[Default]}"
DIVIDER . lightCyan ${PRI_WIDTH}
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[Yellow]}Calculating Air Conditioner Averages ... "
AIRCON_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['ServerRoom-AC']} ${avgdays} both)
AIRCON_AMP_avg=$(echo ${AIRCON_avg} | cut -d'~' -f 1)
AIRCON_WATT_avg=$(echo ${AIRCON_avg} | cut -d'~' -f 2)
echo -en "\r\033[K"
fi
echo -en "${idsCL[White]}Air Conditioner Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${AIRCON_AMP}`'amps ${idsCL[Default]}"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${AIRCON_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[White]}Air Conditioner Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${AIRCON_WATT} 0`'watts ${idsCL[Default]}"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${AIRCON_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}"
DIVIDER . lightCyan ${PRI_WIDTH}
fi
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[Yellow]}Calculating Server Room Total Averages ... "
SVRRM_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total']} ${avgdays} both)
SVRRM_AMP_avg=$(echo ${SVRRM_avg} | cut -d'~' -f 1)
SVRRM_WATT_avg=$(echo ${SVRRM_avg} | cut -d'~' -f 2)
SVRRM_WAC_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total-wac']} ${avgdays} both)
SVRRM_AMP_WAC_avg=$(echo ${SVRRM_WAC_avg} | cut -d'~' -f 1)
SVRRM_WATT_WAC_avg=$(echo ${SVRRM_WAC_avg} | cut -d'~' -f 2)
echo -en "\r\033[K"
fi
echo -e "${idsCL[White]}Server Room Voltage IN ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTIN}'volts${idsCL[Default]}"
echo -e "${idsCL[White]}Server Room Voltage OUT ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTOUT}'volts${idsCL[Default]}"
[ ${REPORT} -eq 0 ] && echo -e "${idsCL[White]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}"
echo -en "${idsCL[White]}Server Room Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[LightCyan]} Total with AirCond ${idsCL[Default]}: ${idsCL[LightCyan]}${SVRRM_AMP_WAC}'amps"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_AMP_WAC_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[White]}Server Room Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${SVRRM_WATT} 0`'watts"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[LightCyan]} Total with AirCond ${idsCL[Default]}: ${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${SVRRM_WATT_WAC} 0`'watts"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_WATT_WAC_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}"
[ ${REPORT} -eq 0 ] && echo
fi
if ([ "${STYPE}" = "" ] || [ "${STYPE}" = "offsite" ]) && [ "${PTYPE}" != "esxi" ]; then
DIVIDER . yellow ${PRI_WIDTH}
echo -e "${idsCL[Yellow]}Offsite Power Information ${idsCL[LightYellow]}${avgdayshdr}"
DIVIDER . yellow ${PRI_WIDTH}
[ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'Offsite' ... "
# upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_OFFSITE_MONITOR} 'upsc CP1500PFCLCD' 2>&1)
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
(( $(bc <<<"${OFFSITE_AMPS} < 1") )) && [[ "${OFFSITE_AMPS}" = *"."* ]] && OFFSITE_AMPS="0${OFFSITE_AMPS}"
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
fi
[ ${REPORT} -eq 0 ] && echo -en "\r\033[K"
if [ "${avgdays}" != "" ]; then
echo -en "${idsCL[Yellow]}Calculating Offsite Averages ... "
OFFSITE_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['offsite-ups']} ${avgdays} both)
OFFSITE_AMP_avg=$(echo ${OFFSITE_avg} | cut -d'~' -f 1)
OFFSITE_WATT_avg=$(echo ${OFFSITE_avg} | cut -d'~' -f 2)
echo -en "\r\033[K"
fi
echo -e "${idsCL[White]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTIN}`'volts${idsCL[Default]}"
echo -e "${idsCL[White]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTOUT}`'volts${idsCL[Default]}"
[ ${REPORT} -eq 0 ] && echo
echo -e "${idsCL[White]}Battery Capacity ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_BATT_CAP}`%${idsCL[Default]}"
echo -e "${idsCL[White]}Battery Runtime ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME}'Mins${idsCL[Default]}"
[ ${REPORT} -eq 0 ] && echo
echo -en "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_AMPS}`'amps ${idsCL[Default]}~ est${idsCL[Green]}${idsCL[Default]}"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${OFFSITE_AMP_avg}`'amps" || echo -e "${idsCL[Default]}"
echo -en "${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_WATTS} 0`'watts ${idsCL[Default]}~ est"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${OFFSITE_WATT_avg} 0`'watts" || echo -e "${idsCL[Default]}\n"
fi
if [ "${PTYPE}" != "power" ]; then
DIVIDER . yellow ${PRI_WIDTH}
echo -e "${idsCL[Yellow]}ESXi Host Power Information ${avgdayshdr}"
DIVIDER . yellow ${PRI_WIDTH}
SENSOR_TOTAL_WATTS_MAINSITE=0
SENSOR_TOTAL_AMPS_MAINSITE=0
SENSOR_TOTAL_WATTS_OFFSITE=0
SENSOR_TOTAL_AMPS_OFFSITE=0
if [ "${avgdays}" != "" ]; then
AVERAGE_WATTS_MAINSITE=0
AVERAGE_AMPS_MAINSITE=0
AVERAGE_WATTS_OFFSITE=0
AVERAGE_AMPS_OFFSITE=0
fi
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then
if [ "${STYPE}" = "" ] || ([ "${STYPE}" = "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${STYPE}" = "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then
[ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... "
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq 2> /dev/null | sed -e 's/"//g')
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
PS1='0.0'; PS2='0.0'; HOST_WATTS=0
if [ "${HOST_WATT_SENSORS}" != "" ]; then
found=0
for s in {1..5}; do
DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')
if [[ "${DP}" = *"System Board"* ]]; then
found=1
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS1"* ]]; then
PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS2"* ]]; then
PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
fi
done
PS1V=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.12.1.16.1.1 | sed 's/.*: //')
PS2V=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.12.1.16.1.2 | sed 's/.*: //')
if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ]; then
[ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}"
[ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}"
HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc`
HOST_WATTS_disp="${idsCL[Green]}${HOST_AMPS}'amps - `IDS_NUMBER_FORMAT ${HOST_WATTS} 0`'watts"
elif [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep off)" != "" ]; then
HOST_WATTS_disp="${idsCL[LightRed]}Host powered down"
HOST_WATTS=-1
elif [ "${HOST_WATTS}" = "0" ]; then
HOST_WATTS_disp="${idsCL[Yellow]}Host PSU not capable"
HOST_WATTS=-2
else
HOST_WATTS_disp="${idsCL[LightRed]}Could not pull data"
fi
[ "${HOST_WATTS}" = "" ] && HOST_WATTS=0
else
HOST_WATTS_disp="${idsCL[LightRed]}Host offline"
fi
MSG="${idsCL[White]}${SENSOR} ${idsCL[Default]}: ${HOST_WATTS_disp}"
[ ${REPORT} -eq 0 ] && echo -en "\r\033[K"
echo -en "${MSG}"
if [ "${HOST_WATT_SENSORS}" != "" ] && [ ${HOST_WATTS} -gt 0 ]; then
if [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]; then
SENSOR_TOTAL_WATTS_MAINSITE=$(expr ${SENSOR_TOTAL_WATTS_MAINSITE} + ${HOST_WATTS})
SENSOR_TOTAL_AMPS_MAINSITE=$(echo "scale=1; ${SENSOR_TOTAL_AMPS_MAINSITE}+${HOST_AMPS}" | bc)
elif [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]; then
SENSOR_TOTAL_WATTS_OFFSITE=$(expr ${SENSOR_TOTAL_WATTS_OFFSITE} + ${HOST_WATTS})
SENSOR_TOTAL_AMPS_OFFSITE=$(echo "scale=1; ${SENSOR_TOTAL_AMPS_OFFSITE}+${HOST_AMPS}" | bc)
fi
fi
# if [ ${HOST_WATTS} -gt 0 ]; then
if [ "${avgdays}" != "" ] && [ ${HOST_WATTS} -ne -2 ] && [ "${SENSOR}" != "iDS-vMS-Host6" ]; then
echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}"
# AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays} both
average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays} both)
average_amp=$(echo ${average} | cut -d'~' -f 1)
average_watt=$(echo ${average} | cut -d'~' -f 2)
average_disp="${idsCL[LightYellow]} ${average_amp}'amps - `IDS_NUMBER_FORMAT ${average_watt} 0`'watts"
echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}"
if [[ "${PW_MAINSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]]; then
AVERAGE_WATTS_MAINSITE=`echo "scale=2; ${AVERAGE_WATTS_MAINSITE}+${average_watt}" | bc`
AVERAGE_AMPS_MAINSITE=`echo "scale=2; ${AVERAGE_AMPS_MAINSITE}+${average_amp}" | bc`
else
AVERAGE_WATTS_OFFSITE=`echo "scale=2; ${AVERAGE_WATTS_OFFSITE}+${average_watt}" | bc`
AVERAGE_AMPS_OFFSITE=`echo "scale=2; ${AVERAGE_AMPS_OFFSITE}+${average_amp}" | bc`
fi
elif [ ${HOST_WATTS} -gt 0 ]; then
echo -en "${idsCL[Cyan]} [PS1:${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}-${idsCL[LightCyan]}${PS1V}'V${idsCL[Cyan]}, PS2:${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}-${idsCL[LightCyan]}${PS2V}'V${idsCL[Cyan]}]"
fi
# fi
echo -e "${idsCL[Default]}"
fi
fi
done
DIVIDER . lightCyan ${PRI_WIDTH}
if [ "${STYPE}" = "" ] || [ "${STYPE}" = "mainsite" ]; then
echo -en "${idsCL[LightCyan]}Mainsite Total ${idsCL[Default]}: ${idsCL[LightGreen]}${SENSOR_TOTAL_AMPS_MAINSITE}'amps - `IDS_NUMBER_FORMAT ${SENSOR_TOTAL_WATTS_MAINSITE} 0`'watts"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} ${AVERAGE_AMPS_MAINSITE}'amps - `IDS_NUMBER_FORMAT ${AVERAGE_WATTS_MAINSITE} 0`'watts" || echo
fi
if [ "${STYPE}" = "" ] || [ "${STYPE}" = "offsite" ]; then
echo -en "${idsCL[LightCyan]}Offsite Total ${idsCL[Default]}: ${idsCL[LightGreen]}${SENSOR_TOTAL_AMPS_OFFSITE}'amps - `IDS_NUMBER_FORMAT ${SENSOR_TOTAL_WATTS_OFFSITE} 0`'watts"
[ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} ${AVERAGE_AMPS_OFFSITE}'amps - `IDS_NUMBER_FORMAT ${AVERAGE_WATTS_OFFSITE} 0`'watts" || echo
fi
fi
if [ ${REPORT} -eq 0 ]; then
# end=`date +%s`; runtime=$((end-start)); echo -e "\n${idsCL[Default]}runtime: ${runtime}s"
[ "${PW_ACTION}" != "" ] && echo -e "${idsCL[Default]}"
fi
}
CHECKPOWER_SERVICE(){
touch ${PW_TMPFOLDER}/power.ac.start
touch ${PW_TMPFOLDER}/power.dc.start
runEvery=10 #SECONDS
while true; do
start=$(date +%s)
source ${PW_FOLDER}/defaults.inc
[ "${TEST_INPUTACV}" != "" ] || [ "${TEST_BATTVOLT}" != "" ] && TESTMODE=1 || TESTMODE=0
INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json")
SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //')
if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then
if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then
touch ${PW_TMPFOLDER}/power.ups.low
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then
SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1
######################### FINISH
fi
elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then
rm -f ${PW_TMPFOLDER}/power.ups.low
fi
if [ "${INVSTATSINFO}" != "" ] || [ ${TESTMODE} -eq 1 ]; then
[ "${TEST_INPUTACV}" != "" ] && INPUTACV=${TEST_INPUTACV} || INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
[ "${TEST_BATTVOLT}" != "" ] && BATTVOLT=${TEST_BATTVOLT} || BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc`
if (( $(bc <<<"${BATTV} >= ${FLOAT_CHARGE}") )); then
if (( $(bc <<<"${BATTV} >= ${BULK_CHARGE}") )); then
BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Bulk Charging"
else
BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Float Charging"
fi
elif [ "${BATTA}" = "-0.1" ] || [ "${BATTA}" = "-0.2" ] || [ "${BATTA}" = "-0.3" ]; then
BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Standby"
else
BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Discharging"
fi
if [ ${TESTMODE} -eq 1 ] && [ ! -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then
touch ${PW_TMPFOLDER}/power.testmode.enabled
SENDNOTICE "TEST MODE" "***TEST MODE*** ENABLED!"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGPOWER}
elif [ ${TESTMODE} -eq 0 ] && [ -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then
rm -f ${PW_TMPFOLDER}/power.testmode.enabled
SENDNOTICE "TEST MODE" "***TEST MODE*** DISABLED!"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGPOWER}
fi
[ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT}
if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then
[ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT}
if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER}
SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected
GRID VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)
BATT AMPS: ${BATTA_disp}"
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then
SENDNOTICE "POWER Startup - Nominal" "Service Startup
Normal voltage detected
GRID VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)
BATT AMPS: ${BATTA_disp}"
echo "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Power' Service Startup - Normal voltage detected - Input: ${INPUTACV}'volts" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.ac.start
fi
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then
if [ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then
SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
touch ${PW_TMPFOLDER}/.power.restored
fi
rm -f ${PW_TMPFOLDER}/power.ac.*
touch ${PW_TMPFOLDER}/power.ac.good
elif [ -f ${PW_TMPFOLDER}/.power.restored ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.power.restored)) -ge $(echo "scale=0; ${PW_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
HOSTMGMT mainsite -p on >/dev/null 2>&1 &
rm -f ${PW_TMPFOLDER}/.power.restored
fi
# UPDATE APC-PDU LINE TO LINE VOLTAGE
CURRENT_VOLTAGE=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //')
SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'`
[ "${SET_VOLTAGE}" != "" ] && [ "${CURRENT_VOLTAGE}" != "" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER}
else
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then
SENDNOTICE "POWER Startup - POWER OFF/LOW!!" "Service Startup
Power off or low voltage detected
GRID VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)
BATT AMPS: ${BATTA_disp}" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Power' Service Startup - Normal voltage detected - Input: ${INPUTACV}'volts" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.ac.start
rm -f ${PW_TMPFOLDER}/power.ac.good
touch ${PW_TMPFOLDER}/power.ac.low
elif [ -f ${PW_TMPFOLDER}/power.ac.good ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected
GRID VOLTAGE: ${INPUTACV}'volts
BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)
BATT AMPS: ${BATTA_disp}" 1
rm -f ${PW_TMPFOLDER}/power.ac.good
touch ${PW_TMPFOLDER}/power.ac.low
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60/1" | bc) ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
# SHUTDOWN_SERVER ${PW_MAIN_ISCSI} -esxi off &
CONSERVE_POWER &
touch ${PW_TMPFOLDER}/power.ac.conserve
fi
fi
if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER}
if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
rm -f ${PW_TMPFOLDER}/power.dc.start
fi
if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then
rm -f ${PW_TMPFOLDER}/power.dc.*
touch ${PW_TMPFOLDER}/power.dc.good
fi
else
[ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good
[ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_ABNORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow
if [ ! -f ${PW_TMPFOLDER}/shutdown.SYS.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.critlow)) -ge 30 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts
Shutting down all remaining servers" 1
SHUTDOWN_SYS POWER &
fi
elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then
[ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low
[ -f ${PW_TMPFOLDER}/power.dc.critlow ] && rm -f ${PW_TMPFOLDER}/power.dc.critlow
if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.low)) -ge 30 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
fi
elif [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_SHUTDOWN_MAINSERVERS_AFTER}*60" | bc) ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts
Shutting down main servers" 1
SHUTDOWN_MAIN POWER &
fi
fi
if [ -f ${PW_TMPFOLDER}/power.ac.low ] || [ -f ${PW_TMPFOLDER}/power.dc.low ]; then
volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc`
(( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=2; ${volt_diff} * -1" | bc`
if (( $(bc <<<"${volt_diff} >= 0.5") )); then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER}
last_battvolt=${BATTVOLT}
fi
fi
else
sleep 20s
fi
end=`date +%s`; runtime=$((end-start))
# echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s"
[ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s ###|| echo "Not sleeping, ran for ${runtime}s previsoulsy"
done # &
}
OFFSITE_CHECKPOWER_SERVICE(){
startup=true
runEvery=10 #SECONDS
while true; do
start=$(date +%s)
source ${PW_FOLDER}/defaults.inc
PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ')
# TEST=true
# TEST=full
if [ "${TEST}" != "" ]; then
PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD=95 #percent
PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes
PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes
TESTMODE="### TEST MODE ACTIVE ###
"
else
TESTMODE=""
fi
# upsinfo=$(upsc CP1500PFCLCD)
upsinfo=$(upsc CP1500PFCLCD 2>&1)
if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then
rm -f ${PW_TMPFOLDER}/power.ups.losscomm
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
if [ ! -d /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']} ]; then
/bin/dtoverlay w1-gpio gpiopin=4
sleep 2s
fi
if [ -d /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']} ]; then
temp_c=$(cat /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']}/w1_slave | tail -n1 | cut -d "=" -f2)
OFFSITE_TEMP=$(echo "scale=2; ${temp_c}/1000" | bc)
else
OFFSITE_TEMP=ERR
fi
SENDNOTICE2(){
MESSAGE="${TESTMODE}${2}
BATTERY CHARGE: ${OFFSITE_BATT_CAP}'%
BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'mins
AC VOLTAGE: ${OFFSITE_VOLTIN}'volts
WATT USAGE: ${OFFSITE_WATTS}'watts
AMBIENT TEMP: ${OFFSITE_TEMP}'F"
[ "${3}" != "" ] && PRIORITY=${3} || PRIORITY=0
SENDNOTICE "${1}" "${MESSAGE}" ${PRIORITY} cosmic
}
if [ "${startup}" = "true" ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE}
OFFSITE_SENDPOWER "OFFSITE POWER Service Startup" &
startup=false
fi
if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then
if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE}
SENDNOTICE2 "OFFSITE POWER" "Batteries are fully recharged"
rm -f ${PW_TMPFOLDER}/power.offsite.charging
# elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then
elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then
# touch ${PW_TMPFOLDER}/power.offsite.charging
echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging
fi
if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE}
SENDNOTICE2 "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
fi
rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
fi
if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
touch ${PW_TMPFOLDER}/power.offsite.restored
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE}
SENDNOTICE2 "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage"
for host in ${PW_OFFSITEHOSTS_IDRAC}; do
if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then
ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
else
echo "TESTMODE: Powering up '${host}'"
fi
done
rm -f ${PW_TMPFOLDER}/power.offsite.restored
rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE
fi
fi
logtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER}
fi
else
[ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low
if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE}
SENDNOTICE2 "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1
touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice
fi
if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE}
SENDNOTICE2 "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
touch ${PW_TMPFOLDER}/shutdown.OFFSITE
for host in ${PW_OFFSITEHOSTS}; do
if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then
${PW_FOLDER}/esxi-scripts/offsite-host-shutdown.ps1 ${host} >/dev/null 2>&1 &
fi
done
fi
if [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER}
fi
fi
else
if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then
touch ${PW_TMPFOLDER}/power.ups.losscomm
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE}
SENDNOTICE2 "NUT DRIVER FAILURE" "OMG WHAT DO WE DO, FIX ME!!! I used to be cool and say..... Restarting NUT service"
# /usr/local/etc/rc.d/nut.sh restart
# sleep 10s
touch ${PW_TMPFOLDER}/power.ups.losscomm
fi
fi
end=`date +%s`; runtime=$((end-start))
# echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s"
[ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s ###|| echo "Not sleeping, ran for ${runtime}s previsoulsy"
done
}
POWERLOGGER_SERVICE(){
SENDPOWER "POWER LOGGING Service Startup" &
while true; do
LogDataTime=`date +'%Y-%m-%d %H:%M:%S'`
source ${PW_FOLDER}/defaults.inc
INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['inv-in']}','${LogDataTime}','${INPUTV}','${INPUTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['inv-out']}','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['battery']}','${LogDataTime}','${BATTV}','${BATTA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then
SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')
SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc`
SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //')
SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //')
SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total']}','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then
APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //')
APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc`
if [ "${SVRRM_VOLT}" != "" ]; then
APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['server-rack']}','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')"
else
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')"
fi
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then
NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc`
NETRK_WATT=$(ROUND_NUMBER `echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc` 0)
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['network-rack']}','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
AIRCON_INFO=$(curl -m 3 -s http://${PW_REMOTE_SENSOR['ServerRoom-AC']}:8888/status/${PW_SENSOR_DEVICEID['ServerRoom-AC']} | jq '.dps')
if [ "${AIRCON_INFO}" != "" ]; then
AIRCON_AMP=$(echo "${AIRCON_INFO}" | grep '"18":' | cut -d' ' -f 4 | sed 's/,//g')
AIRCON_AMP=$(ROUND_NUMBER $(echo "scale=3; ${AIRCON_AMP}/1000" | bc -l) 2)
AIRCON_WATT=$(echo "${AIRCON_INFO}" | grep '"19":' | cut -d' ' -f 4 | sed 's/,//g')
AIRCON_WATT=$(echo "scale=1; ${AIRCON_WATT}/10" | bc -l)
AIRCON_VOLT=$(echo "${AIRCON_INFO}" | grep '"20":' | cut -d' ' -f 4 | sed 's/,//g')
AIRCON_VOLT=$(echo "scale=3; ${AIRCON_VOLT}/10" | bc -l)
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['ServerRoom-AC']}','${LogDataTime}','${AIRCON_VOLT}','${AIRCON_AMP}','${AIRCON_WATT}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
if [ "${SVRRM_WATT}" != "" ]; then
SVRRM_AMP_WAC=$(echo "scale=2; ${SVRRM_AMP}+${AIRCON_AMP}" | bc -l)
SVRRM_WATT_WAC=$(echo "scale=2; ${SVRRM_WATT}+${AIRCON_WATT}" | bc -l)
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total-wac']}','${LogDataTime}','${SVRRM_AMP_WAC}','${SVRRM_WATT_WAC}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
fi
# upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD')
upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_OFFSITE_MONITOR} 'upsc CP1500PFCLCD' 2>&1)
if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then
OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //')
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['offsite-ups']}','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
for SENSOR in ${PW_SENSOR_ORD[@]}; do
if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ] && [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g')
if [[ "${HOST_WATTS}" != *"Timeout"* ]] && [ "${HOST_WATT_SENSORS}" != "" ]; then
IFS=$'\n'
read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}"
unset IFS
[ -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.disable
[ -f ${PW_TMPFOLDER}/shutdown.${PW_HOST_ADDRESSES[${SENSOR}]} ] && rm -f ${PW_TMPFOLDER}/shutdown.${PW_HOST_ADDRESSES[${SENSOR}]}
[ -f ${PW_TMPFOLDER}/shutdown.ALL ] && rm -f ${PW_TMPFOLDER}/shutdown.ALL
[[ "${PW_OFFSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE
[[ "${PW_MAINSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.MAINSITE
[[ "${PW_MAIN_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.MAIN
[[ "${PW_SYS_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.SYS
PS1='0.0'; PS2='0.0'; HOST_WATTS=0
for s in {1..5}; do
DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //')
if [[ "${DP}" = *"System Board"* ]]; then
found=1
HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS1"* ]]; then
PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
elif [[ "${DP}" = *"PS2"* ]]; then
PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //')
fi
done
if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ] && [[ "${HOST_WATTS}" != *"Timeout"* ]]; then
[ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}"
[ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}"
HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID[${SENSOR}]}','${LogDataTime}','${HOST_AMPS}','${HOST_WATTS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
fi
fi
fi
done
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*.low")" != "" ]; then
sleep ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}m
else
sleep ${PW_MYSQL_LOG_INTERVAL_NORMAL}m
fi
done
}
HOSTPOWER(){
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
idracip=${1}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
tmp=${PW_ESXI_HOST_NAMES[${1}]}
idracip=${PW_HOST_IDRACS[${tmp}]}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]}
elif ([ "${1::1}" = "o" ] || [ "${1::1}" = "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${1: -1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[OFF-vMS-Host${1: -1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
idracip=${PW_HOST_IDRACS[${1}]}
else
idracip=''
fi
echo -e "\n${idsCL[Yellow]}POWERING ${2^^} '${PW_iDRACHOST_SHORTNAMES[${idracip}]}'"
DIVIDER . yellow ${PRI_WIDTH}
if [ "${idracip}" != "" ]; then
if [ "${2^^}" = "ON" ] || [ "${2^^}" = "OFF" ] || [ "${2^^}" = "STATUS" ]; then
if [ "${idracip}" = "10.10.2.20" ] && [ "${2^^}" = "ON" ]; then
MSG="Powering ON the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "\n${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
current_status=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 | sed 's/.*: //')
if [ "${current_status}" != "" ] && [ "${current_status}" != "1" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
elif [ "${current_status}" = "1" ]; then
echo -e "${idsCL[Green]}SAS Enclosure Already Powered ON${idsCL[Default]}\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
if [ "${TEST}" != "true" ]; then
current_status=$(ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep "Power is ${2,,}")
if [ "${current_status}" = "" ]; then
echo -e "${idsCL[Green]}"
ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,}
else
echo -e "\n${idsCL[LightCyan]}Host is already Powered ${2^^}${idsCL[Default]}"
fi
else
echo -e "\n${idsCL[Yellow]}(TEST MODE)"
fi
echo -e "${idsCL[Default]}"
if [ "${idracip}" = "10.10.2.20" ] && [ "${2^^}" = "OFF" ]; then
MSG="Powering OFF the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "$(CHECK_HOST 10.10.2.10)" = "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
else
echo -e "${idsCL[LightRed]}'${2^^}' command is not recognized${idsCL[Default]}"
fi
else
echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}"
fi
echo -e "${idsCL[Default]}\n"
}
PDUCONTROL(){
function exitspacing {
echo -e "\n\033[K\n\033[K"
exit 0
}
trap exitspacing EXIT
while [ $# -gt 0 ]; do
case "$1" in
-port) PORT_SEL=${2};;
esac
shift 1
done
echo
DIVIDER . yellow ${PRI_WIDTH}
echo -e "${idsCL[Yellow]}APC-PDU Power Control"
DIVIDER true yellow ${PRI_WIDTH}
until [ "${PORTSEL^^}" = "E" ]; do
[ "${PORT_SEL}" != "" ] && PORTSEL=${PORT_SEL}
if [ "${PORTSEL}" = "" ]; then
PORT_DESC=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2 -O vq 2> /dev/null | sed -e 's/"//g')
PORT_POWER=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4 -O vq 2> /dev/null | sed -e 's/"//g')
IFS=$'\n'
read -rd '' -a PORT_DESC <<<"${PORT_DESC}"
read -rd '' -a PORT_POWER <<<"${PORT_POWER}"
unset IFS
for p in "${!PORT_DESC[@]}"; do
port=$(expr ${p} + 1)
[ ${PORT_POWER[${p}]} -eq 1 ] && portpower_disp="${idsCL[Green]}ON" || portpower_disp="${idsCL[LightRed]}OFF"
[ ${port} -lt 10 ] && pp=" ${port}" || pp=${port}
c=0; spc=''; spc1=`expr 28 - ${#PORT_DESC[${p}]}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done
echo -e "${pp}) ${idsCL[LightCyan]}${PORT_DESC[${p}]}${spc}${idsCL[Default]}: ${portpower_disp}${idsCL[Default]}"
done
echo -e ""
echo -e ""
echo -e "(${idsCL[Yellow]}E${idsCL[Default]})xit"
echo -e ""
echo -e "\033[5A"
# until [ "${PORTSEL}" != "" ]; do
echo -en "\n${idsCL[LightCyan]}Enter the port number to control: ${idsCL[Default]}"
read PORTSEL
echo
# done
fi
[ "${PORTSEL^^}" = "E" ] && exit 0
[ "${PORTSEL}" != "" ] && tp=$(expr ${PORTSEL} - 1)
if [ "${PORTSEL}" != "" ] && [[ "${PORTSEL}" =~ ^[0-9]+$ ]] && [ "${PORT_DESC[${tp}]}" != "" ] && [ ${PORTSEL} -ne 0 ]; then
# portsel_power=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g')
[ ${PORT_POWER[${tp}]} -eq 1 ] && portsel_power_disp="${idsCL[Green]}ON" || portsel_power_disp="${idsCL[LightRed]}OFF"
[ ${PORT_POWER[${tp}]} -eq 2 ] && portsel_change_power_disp="${idsCL[Green]}ON" || portsel_change_power_disp="${idsCL[LightRed]}OFF"
# portdesc=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g')
DIVIDER . yellow 60
echo -e "${idsCL[Yellow]}Port ${idsCL[LightYellow]}${PORTSEL}${idsCL[Default]}) ${idsCL[LightCyan]}${PORT_DESC[${tp}]} ${idsCL[Default]}: ${portsel_power_disp}${idsCL[Default]}\n"
echo -en "${idsCL[LightCyan]}Are you sure you wish to turn ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} (y/N): "
read -n 1 portchange_confirm
case "${portchange_confirm}" in
[Yy])
[ ${PORT_POWER[${tp}]} -eq 1 ] && portsel_change=2 || portsel_change=1
echo -en "\n\n${idsCL[LightCyan]}Turning ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} ... "
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} i ${portsel_change} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo
;;
esac
[ "${PORT_SEL}" = "" ] && unset PORTSEL
else
echo -e "\n${idsCL[LightRed]}Not a valid port number${idsCL[Default]}"
unset PORTSEL
fi
[ "${PORT_SEL}" != "" ] && exit 0 || echo
done
}
SENDPOWER(){
st_power=`${PW_SCRIPT} check power -m -p -r`
SENDNOTICE "${1}" "${st_power}"
}
OFFSITE_SENDPOWER(){
st_power=`${PW_SCRIPT} check power -o -p -r`
SENDNOTICE "${1}" "${st_power}" 0 cosmic
}