diff --git a/inc/power.inc.bak.sh b/inc/power.inc.bak.sh deleted file mode 100644 index 4d1221d4..00000000 --- a/inc/power.inc.bak.sh +++ /dev/null @@ -1,1090 +0,0 @@ -AVERAGEPOWER(){ - sensorid=${1} - daysback=${2} - - if [[ "${daysback^^}" = *"M"* ]]; then - interval="${daysback::-1} MINUTE" - elif [[ "${daysback^^}" = *"H"* ]]; then - interval="${daysback::-1} HOUR" - elif [[ "${daysback^^}" = *"D"* ]]; then - interval="${daysback::-1} DAY" - elif [[ "${daysback}" =~ ^[0-9]+$ ]]; then - interval="${daysback} DAY" - fi - - if [ "${3}" = "both" ]; then - logsfound_amp=0; - logsfound_watt=0; - total_amp=0 - total_watt=0 - while IFS=$'\t' read date amp watt ;do - if [ "${amp}" != "" ] && [ "${amp}" != "NULL" ]; then - total_amp=$(bc <<< "scale=2; ${total_amp}+${amp}") - ((logsfound_amp++)) - fi - if [ "${watt}" != "" ] && [ "${watt}" != "NULL" ]; then - total_watt=$(bc <<< "scale=2; ${total_watt}+${watt}") - ((logsfound_watt++)) - fi - done < <(${mysql_conn} -se "USE servermonitor; SELECT date,amp,watt from power_data WHERE sensorid=${sensorid} AND date >= DATE_SUB(NOW(), INTERVAL ${interval}) ORDER BY id DESC;") - if [ "${logsfound_watt}" != "0" ]; then - average_amp=$(bc <<< "scale=2; ${total_amp}/${logsfound_amp}") - average_watt=$(bc <<< "scale=2; ${total_watt}/${logsfound_watt}") - else - average_amp=0 - average_watt=0 - fi - echo "${average_amp}~${average_watt}" - else - logsfound=0; - total=0 - [ "${3}" != "" ] && ptype=${3} || ptype=watt - while IFS=$'\t' read date ${ptype} ;do - if [ "${!ptype}" != "" ] && [ "${!ptype}" != "NULL" ]; then - total=$(bc <<< "scale=2; ${total}+${!ptype}") - ((logsfound++)) - fi - done < <(${mysql_conn} -se "USE servermonitor; SELECT date,${ptype} from power_data WHERE sensorid=${sensorid} AND date >= DATE_SUB(NOW(), INTERVAL ${interval}) ORDER BY id DESC;") - [ "${logsfound}" != "0" ] && average=$(bc <<< "scale=2; ${total}/${logsfound}") || average=0 - echo ${average} - fi -} - -CHECKPOWER(){ - start=`date +%s` - rm -f ${PW_TMPFOLDER}/*.hostpower* - REPORT=0 - while [ $# -gt 0 ]; do - case "$1" in - -a|-avg) avgdays=${2};; - -o|-offsite) STYPE=offsite;; - -m|-mainsite) STYPE=mainsite;; - -e|-esxi) PTYPE=esxi;; - -p|-power) PTYPE=power;; - -r|-report) - REPORT=1 - unset idsCL idsBG idsST - idsCL=('') - idsBG=('') - idsST=('') - ;; - esac - shift 1 - done - if [ "${avgdays}" != "" ]; then - if [ "${avgdays}" = "0" ]; then - unset avgdays - elif [[ "${avgdays^^}" = *"M"* ]]; then - avgdays_disp="${avgdays::-1}Min" - elif [[ "${avgdays^^}" = *"H"* ]]; then - avgdays_disp="${avgdays::-1}Hr" - elif [[ "${avgdays^^}" = *"D"* ]]; then - avgdays_disp="${avgdays::-1}Day" - elif [[ "${avgdays}" =~ ^[0-9]+$ ]]; then - avgdays_disp="${avgdays}Day" - fi - # else - # avgdays=noavg - fi - - if ([ "${STYPE}" = "" ] || [ "${STYPE}" = "mainsite" ]) && [ "${PTYPE}" != "esxi" ]; then - [ ${REPORT} -eq 0 ] && echo - [ ${REPORT} -eq 0 ] && DIVIDER . yellow ${PRI_WIDTH} - echo -en "${idsCL[Yellow]}Mainsite Power Information" - [ "${avgdays}" != "" ] && echo -e " ${idsCL[LightYellow]}${avgdays_disp}-AVG" || echo - DIVIDER . yellow ${PRI_WIDTH} - - [ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'Powerwall' ... " - - INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json") - if [ "${INVSTATSINFO}" != "" ]; then - [ "${TEST_INPUTACV}" != "" ] && INPUTV=${TEST_INPUTACV} || INPUTV=$(echo ${INVSTATSINFO} | jq '.inputs .inV') - INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'` - INV_ALARM=`echo ${INVSTATSINFO} | jq '.errors .Alms'` - [ "${INV_ALARM}" = "0" ] && INV_ALARM_disp="${idsCL[Green]}No Alarms (${INV_ALARM})" || INV_ALARM_disp="${idsCL[LightRed]}${PW_INVERTER_ALARMS[${INV_ALARM}]}" - INV_ERROR=`echo ${INVSTATSINFO} | jq '.errors .Err'` - [ "${INV_ERROR}" = "0" ] && INV_ERROR_disp="${idsCL[Green]}No Errors (${INV_ERROR})" || INV_ERROR_disp="${idsCL[LightRed]}${PW_INVERTER_ERRORS[${INV_ERROR}]}" - INV_STATE=`echo ${INVSTATSINFO} | jq '.stats .invSTATES'` - [ "${INV_STATE}" != "0" ] && INV_STATE_disp="${idsCL[Green]}${PW_INVERTER_STATES[${INV_STATE}]}" || INV_STATE_disp="${idsCL[LightRed]}${PW_INVERTER_STATES[${INV_STATE}]}" - INV_MODE=`echo ${INVSTATSINFO} | jq '.stats .power'` - ([ "${INV_MODE}" != "3" ] && [ "${INV_MODE}" != "4" ] && [ "${INV_MODE}" != "E" ]) && INV_MODE_disp="${idsCL[Green]}${PW_INVERTER_MODES[${INV_MODE}]}" || INV_MODE_disp="${idsCL[LightRed]}${PW_INVERTER_MODES[${INV_MODE}]}" - OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'` - OUTPUTV2=`echo "scale=2; ${OUTPUTV}/2" | bc` - OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'` - OUTPUTA2=`echo "scale=2; ${OUTPUTA}*2" | bc` - OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'` - [ "${TEST_BATTVOLT}" != "" ] && BATTV=${TEST_BATTVOLT} || BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'` - BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'` - BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc` - if (( $(bc <<<"${BATTV} >= ${FLOAT_CHARGE}") )); then - if (( $(bc <<<"${BATTV} >= ${BULK_CHARGE}") )); then - BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Bulk Charging" - else - BATTA_disp="${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Float Charging" - fi - elif [ "${BATTA}" = "-0.1" ] || [ "${BATTA}" = "-0.2" ] || [ "${BATTA}" = "-0.3" ]; then - BATTA_disp="${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Standby" - else - BATTA_disp="${idsCL[LightRed]}`IDS_NUMBER_FORMAT ${BATTA}`'amps ${idsCL[LightCyan]}- Discharging" - fi - fi - - [ ${REPORT} -eq 0 ] && echo -en "\r\033[K" - echo -e "${idsCL[White]}Inverter State ${idsCL[Default]}: ${INV_STATE_disp}" - if [ ${REPORT} -eq 0 ]; then - echo -e "${idsCL[White]}Inverter Mode ${idsCL[Default]}: ${INV_MODE_disp}" - echo -e "${idsCL[White]}Inverter Alarms ${idsCL[Default]}: ${INV_ALARM_disp}" - echo -e "${idsCL[White]}Inverter Errors ${idsCL[Default]}: ${INV_ERROR_disp}" - echo - fi - echo -en "${idsCL[White]}Inverter Input Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTV}`'volts" - [ "${TEST_INPUTACV}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo "" - echo -e "${idsCL[White]}Inverter Input Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${INPUTA}`'amps" - if [ ${REPORT} -eq 0 ]; then - echo - echo -e "${idsCL[White]}Inverter Output Volts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTV}`'volts / `IDS_NUMBER_FORMAT ${OUTPUTV2}`'volts" - echo -e "${idsCL[White]}Inverter Output Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTA}`'amps / `IDS_NUMBER_FORMAT ${OUTPUTA2}`'amps" - echo -e "${idsCL[White]}Inverter Output Watts ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OUTPUTW} 0`'watts" - echo - fi - - echo -en "${idsCL[White]}Battery Voltage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${BATTV}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V)" - [ "${TEST_BATTVOLT}" != "" ] && echo -e " ${idsCL[Yellow]}***TEST MODE***" || echo "" - echo -e "${idsCL[White]}Battery Amps Chg/DChg ${idsCL[Default]}: ${BATTA_disp}${idsCL[Default]}" - - # echo - DIVIDER . lightCyan ${PRI_WIDTH} - - [ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'ServerRoom' ... " - - # if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then - SVRRM_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //') - SVRRM_VOLTIN=`echo "scale=2; ${SVRRM_VOLTIN}/10" | bc` - SVRRM_VOLTOUT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //') - SVRRM_VOLTOUT=`echo "scale=2; ${SVRRM_VOLTOUT}/10" | bc` - - SVRRM_WATT=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //') - SVRRM_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //') - SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc` - - AIRCON_INFO=$(curl -m 3 -s http://${PW_REMOTE_SENSORS['ServerRoom-AC']}:8888/status/${PW_SENSOR_DEVICEID['ServerRoom-AC']} | jq '.dps') - AIRCON_AMP=$(echo "${AIRCON_INFO}" | grep '"18":' | cut -d' ' -f 4 | sed 's/,//g') - AIRCON_AMP=$(ROUND_NUMBER $(echo "scale=3; ${AIRCON_AMP}/1000" | bc -l) 2) - AIRCON_WATT=$(echo "${AIRCON_INFO}" | grep '"19":' | cut -d' ' -f 4 | sed 's/,//g') - AIRCON_WATT=$(echo "scale=1; ${AIRCON_WATT}/10" | bc -l) - - SVRRM_AMP_WAC=$(echo "scale=2; ${SVRRM_AMP}+${AIRCON_AMP}" | bc -l) - SVRRM_WATT_WAC=$(echo "scale=2; ${SVRRM_WATT}+${AIRCON_WATT}" | bc -l) - # fi - # - # if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then - APCPDU_AMP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //') - APCPDU_AMP=`echo "scale=2; ${APCPDU_AMP}/10" | bc` - APCPDU_WATT2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.16.0 | sed 's/.*: //') - APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLTOUT}" | bc` - # fi - - if [ "${SVRRM_VOLTOUT}" != "" ]; then - if [ "${APCPDU_AMP}" != "" ]; then - NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc` - NETRK_WATT=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc` - NETRK_WATT2=`echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT2}" | bc` - fi - VOLT_DROP=`echo "scale=1; ${OUTPUTV2}-${SVRRM_VOLTIN}" | bc` - fi - - if [ ${REPORT} -eq 0 ]; then - echo -en "\r\033[K" - if [ "${avgdays}" != "" ]; then - echo -en "${idsCL[Yellow]}Calculating ServerRack Averages ... " - APCPDU_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['server-rack']} ${avgdays} both) - APCPDU_AMP_avg=$(echo ${APCPDU_avg} | cut -d'~' -f 1) - APCPDU_WATT_avg=$(echo ${APCPDU_avg} | cut -d'~' -f 2) - echo -en "\r\033[K" - fi - echo -en "${idsCL[White]}ServerRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_AMP} 2 yes`'amps" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${APCPDU_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[White]}ServerRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${APCPDU_WATT} 0`'watts ${idsCL[Default]}~ est" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${APCPDU_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${APCPDU_WATT2}` watts)${idsCL[Default]}" - - DIVIDER . lightCyan ${PRI_WIDTH} - if [ "${avgdays}" != "" ]; then - echo -en "${idsCL[Yellow]}Calculating NetworkRack Averages ... " - NETRK_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['network-rack']} ${avgdays} both) - NETRK_AMP_avg=$(echo ${NETRK_avg} | cut -d'~' -f 1) - NETRK_WATT_avg=$(echo ${NETRK_avg} | cut -d'~' -f 2) - echo -en "\r\033[K" - fi - echo -en "${idsCL[White]}NetworkRack Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_AMP} 2 yes`'amps ${idsCL[Default]}~ est" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${NETRK_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[White]}NetworkRack Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${NETRK_WATT} 0`'watts ${idsCL[Default]}~ est" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${NETRK_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e " ${idsCL[LightCyan]}(`IDS_NUMBER_FORMAT ${NETRK_WATT2} 0` watts) ${idsCL[Default]}~ est${idsCL[Default]}" - - DIVIDER . lightCyan ${PRI_WIDTH} - if [ "${avgdays}" != "" ]; then - echo -en "${idsCL[Yellow]}Calculating Air Conditioner Averages ... " - AIRCON_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['ServerRoom-AC']} ${avgdays} both) - AIRCON_AMP_avg=$(echo ${AIRCON_avg} | cut -d'~' -f 1) - AIRCON_WATT_avg=$(echo ${AIRCON_avg} | cut -d'~' -f 2) - echo -en "\r\033[K" - fi - echo -en "${idsCL[White]}Air Conditioner Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${AIRCON_AMP}`'amps ${idsCL[Default]}" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${AIRCON_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[White]}Air Conditioner Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${AIRCON_WATT} 0`'watts ${idsCL[Default]}" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${AIRCON_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}" - - DIVIDER . lightCyan ${PRI_WIDTH} - fi - if [ "${avgdays}" != "" ]; then - echo -en "${idsCL[Yellow]}Calculating Server Room Total Averages ... " - SVRRM_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total']} ${avgdays} both) - SVRRM_AMP_avg=$(echo ${SVRRM_avg} | cut -d'~' -f 1) - SVRRM_WATT_avg=$(echo ${SVRRM_avg} | cut -d'~' -f 2) - SVRRM_WAC_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['total-wac']} ${avgdays} both) - SVRRM_AMP_WAC_avg=$(echo ${SVRRM_WAC_avg} | cut -d'~' -f 1) - SVRRM_WATT_WAC_avg=$(echo ${SVRRM_WAC_avg} | cut -d'~' -f 2) - echo -en "\r\033[K" - fi - echo -e "${idsCL[White]}Server Room Voltage IN ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTIN}'volts${idsCL[Default]}" - echo -e "${idsCL[White]}Server Room Voltage OUT ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_VOLTOUT}'volts${idsCL[Default]}" - [ ${REPORT} -eq 0 ] && echo -e "${idsCL[White]}Voltage Drop ${idsCL[Default]}: ${idsCL[LightRed]}${VOLT_DROP}'volts${idsCL[Default]}" - - - echo -en "${idsCL[White]}Server Room Amps ${idsCL[Default]}: ${idsCL[Green]}${SVRRM_AMP}'amps${idsCL[Default]}" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_AMP_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[LightCyan]} Total with AirCond ${idsCL[Default]}: ${idsCL[LightCyan]}${SVRRM_AMP_WAC}'amps" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_AMP_WAC_avg}`'amps${idsCL[Default]}" || echo -e "${idsCL[Default]}" - - echo -en "${idsCL[White]}Server Room Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${SVRRM_WATT} 0`'watts" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_WATT_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[LightCyan]} Total with AirCond ${idsCL[Default]}: ${idsCL[LightCyan]}`IDS_NUMBER_FORMAT ${SVRRM_WATT_WAC} 0`'watts" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${SVRRM_WATT_WAC_avg} 0`'watts${idsCL[Default]}" || echo -e "${idsCL[Default]}" - - [ ${REPORT} -eq 0 ] && echo - - fi - - if ([ "${STYPE}" = "" ] || [ "${STYPE}" = "offsite" ]) && [ "${PTYPE}" != "esxi" ]; then - - DIVIDER . yellow ${PRI_WIDTH} - echo -e "${idsCL[Yellow]}Offsite Power Information ${idsCL[LightYellow]}${avgdayshdr}" - DIVIDER . yellow ${PRI_WIDTH} - - [ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'Offsite' ... " - - # upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') - upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_OFFSITE_MONITOR} 'upsc CP1500PFCLCD' 2>&1) - if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then - OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') - OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //') - OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') - OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` - OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` - OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc` - (( $(bc <<<"${OFFSITE_AMPS} < 1") )) && [[ "${OFFSITE_AMPS}" = *"."* ]] && OFFSITE_AMPS="0${OFFSITE_AMPS}" - OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') - OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` - OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') - - fi - - [ ${REPORT} -eq 0 ] && echo -en "\r\033[K" - if [ "${avgdays}" != "" ]; then - echo -en "${idsCL[Yellow]}Calculating Offsite Averages ... " - OFFSITE_avg=$(AVERAGEPOWER ${PW_POWERSENSOR_ID['offsite-ups']} ${avgdays} both) - OFFSITE_AMP_avg=$(echo ${OFFSITE_avg} | cut -d'~' -f 1) - OFFSITE_WATT_avg=$(echo ${OFFSITE_avg} | cut -d'~' -f 2) - echo -en "\r\033[K" - fi - - echo -e "${idsCL[White]}UPS Voltage In ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTIN}`'volts${idsCL[Default]}" - echo -e "${idsCL[White]}UPS Voltage Out ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_VOLTOUT}`'volts${idsCL[Default]}" - [ ${REPORT} -eq 0 ] && echo - - echo -e "${idsCL[White]}Battery Capacity ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_BATT_CAP}`%${idsCL[Default]}" - echo -e "${idsCL[White]}Battery Runtime ${idsCL[Default]}: ${idsCL[Green]}${OFFSITE_BATT_RUNTIME}'mins${idsCL[Default]}" - [ ${REPORT} -eq 0 ] && echo - - echo -en "${idsCL[White]}Total Amps ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_AMPS}`'amps ${idsCL[Default]}~ est${idsCL[Green]}${idsCL[Default]}" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${OFFSITE_AMP_avg}`'amps" || echo -e "${idsCL[Default]}" - echo -en "${idsCL[White]}Total Wattage ${idsCL[Default]}: ${idsCL[Green]}`IDS_NUMBER_FORMAT ${OFFSITE_WATTS} 0`'watts ${idsCL[Default]}~ est" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${OFFSITE_WATT_avg} 0`'watts" || echo -e "${idsCL[Default]}\n" - - fi - - if [ "${PTYPE}" != "power" ]; then - DIVIDER . yellow ${PRI_WIDTH} - echo -e "${idsCL[Yellow]}ESXi Host Power Information ${avgdayshdr}" - DIVIDER . yellow ${PRI_WIDTH} - SENSOR_TOTAL_WATTS_MAINSITE=0 - SENSOR_TOTAL_AMPS_MAINSITE=0 - SENSOR_TOTAL_WATTS_OFFSITE=0 - SENSOR_TOTAL_AMPS_OFFSITE=0 - if [ "${avgdays}" != "" ]; then - AVERAGE_WATTS_MAINSITE=0 - AVERAGE_AMPS_MAINSITE=0 - AVERAGE_WATTS_OFFSITE=0 - AVERAGE_AMPS_OFFSITE=0 - fi - for SENSOR in ${PW_SENSOR_ORD[@]}; do - if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ]; then - if [ "${STYPE}" = "" ] || ([ "${STYPE}" = "mainsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]) || ([ "${STYPE}" = "offsite" ] && [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]); then - [ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... " - - HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq 2> /dev/null | sed -e 's/"//g') - IFS=$'\n' - read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}" - unset IFS - PS1='0.0'; PS2='0.0'; HOST_WATTS=0 - if [ "${HOST_WATT_SENSORS}" != "" ]; then - found=0 - for s in {1..5}; do - DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //') - if [[ "${DP}" = *"System Board"* ]]; then - found=1 - HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - - elif [[ "${DP}" = *"PS1"* ]]; then - PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - elif [[ "${DP}" = *"PS2"* ]]; then - PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - fi - done - PS1V=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.12.1.16.1.1 | sed 's/.*: //') - PS2V=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.12.1.16.1.2 | sed 's/.*: //') - - if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ]; then - [ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}" - [ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}" - HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc` - HOST_WATTS_disp="${idsCL[Green]}${HOST_AMPS}'amps - `IDS_NUMBER_FORMAT ${HOST_WATTS} 0`'watts" - - elif [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep off)" != "" ]; then - HOST_WATTS_disp="${idsCL[LightRed]}Host powered down" - HOST_WATTS=-1 - elif [ "${HOST_WATTS}" = "0" ]; then - HOST_WATTS_disp="${idsCL[Yellow]}Host PSU not capable" - HOST_WATTS=-2 - else - HOST_WATTS_disp="${idsCL[LightRed]}Could not pull data" - fi - - [ "${HOST_WATTS}" = "" ] && HOST_WATTS=0 - - else - HOST_WATTS_disp="${idsCL[LightRed]}Host offline" - fi - - MSG="${idsCL[White]}${SENSOR} ${idsCL[Default]}: ${HOST_WATTS_disp}" - [ ${REPORT} -eq 0 ] && echo -en "\r\033[K" - echo -en "${MSG}" - if [ "${HOST_WATT_SENSORS}" != "" ] && [ ${HOST_WATTS} -gt 0 ]; then - if [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.10."* ]]; then - SENSOR_TOTAL_WATTS_MAINSITE=$(expr ${SENSOR_TOTAL_WATTS_MAINSITE} + ${HOST_WATTS}) - SENSOR_TOTAL_AMPS_MAINSITE=$(echo "scale=1; ${SENSOR_TOTAL_AMPS_MAINSITE}+${HOST_AMPS}" | bc) - elif [[ "${PW_HOST_IDRACS[${SENSOR}]}" = *"10.2.1."* ]]; then - SENSOR_TOTAL_WATTS_OFFSITE=$(expr ${SENSOR_TOTAL_WATTS_OFFSITE} + ${HOST_WATTS}) - SENSOR_TOTAL_AMPS_OFFSITE=$(echo "scale=1; ${SENSOR_TOTAL_AMPS_OFFSITE}+${HOST_AMPS}" | bc) - fi - fi - # if [ ${HOST_WATTS} -gt 0 ]; then - if [ "${avgdays}" != "" ] && [ ${HOST_WATTS} -ne -2 ] && [ "${SENSOR}" != "iDS-vMS-Host6" ]; then - echo -en "${idsCL[LightYellow]} Calculating average ...${idsCL[Default]}" - # AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays} both - average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays} both) - average_amp=$(echo ${average} | cut -d'~' -f 1) - average_watt=$(echo ${average} | cut -d'~' -f 2) - average_disp="${idsCL[LightYellow]} ${average_amp}'amps - `IDS_NUMBER_FORMAT ${average_watt} 0`'watts" - echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}" - if [[ "${PW_MAINSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]]; then - AVERAGE_WATTS_MAINSITE=`echo "scale=2; ${AVERAGE_WATTS_MAINSITE}+${average_watt}" | bc` - AVERAGE_AMPS_MAINSITE=`echo "scale=2; ${AVERAGE_AMPS_MAINSITE}+${average_amp}" | bc` - else - AVERAGE_WATTS_OFFSITE=`echo "scale=2; ${AVERAGE_WATTS_OFFSITE}+${average_watt}" | bc` - AVERAGE_AMPS_OFFSITE=`echo "scale=2; ${AVERAGE_AMPS_OFFSITE}+${average_amp}" | bc` - fi - - elif [ ${HOST_WATTS} -gt 0 ]; then - echo -en "${idsCL[Cyan]} [PS1:${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}-${idsCL[LightCyan]}${PS1V}'V${idsCL[Cyan]}, PS2:${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}-${idsCL[LightCyan]}${PS2V}'V${idsCL[Cyan]}]" - fi - # fi - echo -e "${idsCL[Default]}" - fi - fi - done - DIVIDER . lightCyan ${PRI_WIDTH} - if [ "${STYPE}" = "" ] || [ "${STYPE}" = "mainsite" ]; then - echo -en "${idsCL[LightCyan]}Mainsite Total ${idsCL[Default]}: ${idsCL[LightGreen]}${SENSOR_TOTAL_AMPS_MAINSITE}'amps - `IDS_NUMBER_FORMAT ${SENSOR_TOTAL_WATTS_MAINSITE} 0`'watts" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} ${AVERAGE_AMPS_MAINSITE}'amps - `IDS_NUMBER_FORMAT ${AVERAGE_WATTS_MAINSITE} 0`'watts" || echo - fi - if [ "${STYPE}" = "" ] || [ "${STYPE}" = "offsite" ]; then - echo -en "${idsCL[LightCyan]}Offsite Total ${idsCL[Default]}: ${idsCL[LightGreen]}${SENSOR_TOTAL_AMPS_OFFSITE}'amps - `IDS_NUMBER_FORMAT ${SENSOR_TOTAL_WATTS_OFFSITE} 0`'watts" - [ "${avgdays}" != "" ] && echo -e "${idsCL[LightYellow]} ${AVERAGE_AMPS_OFFSITE}'amps - `IDS_NUMBER_FORMAT ${AVERAGE_WATTS_OFFSITE} 0`'watts" || echo - fi - fi - if [ ${REPORT} -eq 0 ]; then - # end=`date +%s`; runtime=$((end-start)); echo -e "\n${idsCL[Default]}runtime: ${runtime}s" - [ "${PW_ACTION}" != "" ] && echo -e "${idsCL[Default]}" - fi -} - -CHECKPOWER_SERVICE(){ - touch ${PW_TMPFOLDER}/power.ac.start - touch ${PW_TMPFOLDER}/power.dc.start - runEvery=10 #SECONDS - while true; do - start=$(date +%s) - source ${PW_FOLDER}/defaults.inc - - [ "${TEST_INPUTACV}" != "" ] || [ "${TEST_BATTVOLT}" != "" ] && TESTMODE=1 || TESTMODE=0 - - INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json") - SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //') - - if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then - if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then - touch ${PW_TMPFOLDER}/power.ups.low - - elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then - SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "UPS has no power for 1min, shutting all servers down!" 1 - - - ######################### FINISH - - - fi - - elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then - rm -f ${PW_TMPFOLDER}/power.ups.low - fi - - if [ "${INVSTATSINFO}" != "" ] || [ ${TESTMODE} -eq 1 ]; then - [ "${TEST_INPUTACV}" != "" ] && INPUTACV=${TEST_INPUTACV} || INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV') - [ "${TEST_BATTVOLT}" != "" ] && BATTVOLT=${TEST_BATTVOLT} || BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV') - BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'` - BATTLIFE=`echo "scale=2; (${BATTV}*100)/${BATT_FULL}" | bc` - if (( $(bc <<<"${BATTV} >= ${FLOAT_CHARGE}") )); then - if (( $(bc <<<"${BATTV} >= ${BULK_CHARGE}") )); then - BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Bulk Charging" - else - BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Float Charging" - fi - elif [ "${BATTA}" = "-0.1" ] || [ "${BATTA}" = "-0.2" ] || [ "${BATTA}" = "-0.3" ]; then - BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Standby" - else - BATTA_disp="`IDS_NUMBER_FORMAT ${BATTA}`'amps - Discharging" - fi - - if [ ${TESTMODE} -eq 1 ] && [ ! -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then - touch ${PW_TMPFOLDER}/power.testmode.enabled - SENDNOTICE "TEST MODE" "***TEST MODE*** ENABLED!" - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** ENABLED" >> ${PW_LOGPOWER} - elif [ ${TESTMODE} -eq 0 ] && [ -f ${PW_TMPFOLDER}/power.testmode.enabled ]; then - rm -f ${PW_TMPFOLDER}/power.testmode.enabled - SENDNOTICE "TEST MODE" "***TEST MODE*** DISABLED!" - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Input: ${INPUTACV}'volts - Battery: ${BATTVOLT}'volts - ***TEST MODE*** DISABLED" >> ${PW_LOGPOWER} - fi - - [ ! "${last_battvolt}" ] && last_battvolt=${BATTVOLT} - - if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then - [ "${last_battvolt}" != "${BATTVOLT}" ] && last_battvolt=${BATTVOLT} - if [ -f ${PW_TMPFOLDER}/power.ac.low ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Back to Normal Voltage" >> ${PW_LOGPOWER} - SENDNOTICE "POWER NOMINAL AGAIN" "Normal voltage detected -GRID VOLTAGE: ${INPUTACV}'volts -BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V) -BATT AMPS: ${BATTA_disp}" - else - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts - Normal Voltage" >> ${PW_LOGPOWER} - if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then - SENDNOTICE "POWER Startup - Nominal" "Service Startup -Normal voltage detected -GRID VOLTAGE: ${INPUTACV}'volts -BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V) -BATT AMPS: ${BATTA_disp}" - - echo "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Power' Service Startup - Normal voltage detected - Input: ${INPUTACV}'volts" >> ${PW_LOGPOWER} - rm -f ${PW_TMPFOLDER}/power.ac.start - fi - fi - if [ ! -f ${PW_TMPFOLDER}/power.ac.good ]; then - - if [ -f ${PW_TMPFOLDER}/shutdown.SYS ] || [ -f ${PW_TMPFOLDER}/shutdown.MAIN ] || [ -f ${PW_TMPFOLDER}/power.ac.conserve ]; then - SENDNOTICE "HOSTS POWER" "Power restored! Will wait ${PW_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" - touch ${PW_TMPFOLDER}/.power.restored - fi - - rm -f ${PW_TMPFOLDER}/power.ac.* - touch ${PW_TMPFOLDER}/power.ac.good - - elif [ -f ${PW_TMPFOLDER}/.power.restored ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.power.restored)) -ge $(echo "scale=0; ${PW_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then - HOSTMGMT mainsite -p on >/dev/null 2>&1 & - rm -f ${PW_TMPFOLDER}/.power.restored - fi - - # UPDATE APC-PDU LINE TO LINE VOLTAGE - CURRENT_VOLTAGE=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 | sed 's/.*: //') - SET_VOLTAGE=`echo "scale=2; $(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //')/10" | bc | awk '{print int($1+0.5)}'` - [ "${SET_VOLTAGE}" != "" ] && [ "${CURRENT_VOLTAGE}" != "" ] && [ ${CURRENT_VOLTAGE} -ne ${SET_VOLTAGE} ] && run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.1.15.0 i ${SET_VOLTAGE} >/dev/null 2>&1) && echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${CURRENT_VOLTAGE}'volts --> ${SET_VOLTAGE}'volts - LINE-TO-LINE VOLTAGE UPDATED ON APC-PDU" >> ${PW_LOGPOWER} - - else - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge 300 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW" >> ${PW_LOGPOWER} - if [ -f ${PW_TMPFOLDER}/power.ac.start ]; then - SENDNOTICE "POWER Startup - POWER OFF/LOW!!" "Service Startup -Power off or low voltage detected -GRID VOLTAGE: ${INPUTACV}'volts -BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V) -BATT AMPS: ${BATTA_disp}" 1 - echo "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Power' Service Startup - Normal voltage detected - Input: ${INPUTACV}'volts" >> ${PW_LOGPOWER} - rm -f ${PW_TMPFOLDER}/power.ac.start - rm -f ${PW_TMPFOLDER}/power.ac.good - touch ${PW_TMPFOLDER}/power.ac.low - - elif [ -f ${PW_TMPFOLDER}/power.ac.good ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Power OFF or LOW voltage detected" >> ${PW_LOGPOWER} - SENDNOTICE "POWER ALERT - POWER OFF/LOW!!" "Power off or low voltage detected -GRID VOLTAGE: ${INPUTACV}'volts -BATT VOLTAGE: `IDS_NUMBER_FORMAT ${BATTVOLT}`'volts ~ (`IDS_NUMBER_FORMAT ${BATTLIFE}`% @ ${BATT_FULL}'V) -BATT AMPS: ${BATTA_disp}" 1 - rm -f ${PW_TMPFOLDER}/power.ac.good - touch ${PW_TMPFOLDER}/power.ac.low - - fi - if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60/1" | bc) ]; then - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER} - SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1 - # SHUTDOWN_SERVER ${PW_MAIN_ISCSI} -esxi off & - CONSERVE_POWER & - touch ${PW_TMPFOLDER}/power.ac.conserve - fi - fi - - if (( $(bc <<<"${BATTVOLT} >= ${min_battvolt}") )); then - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage" >> ${PW_LOGPOWER} - if [ -f ${PW_TMPFOLDER}/power.dc.start ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - Battery: ${BATTVOLT}'volts - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER} - rm -f ${PW_TMPFOLDER}/power.dc.start - fi - if [ ! -f ${PW_TMPFOLDER}/power.dc.good ]; then - rm -f ${PW_TMPFOLDER}/power.dc.* - touch ${PW_TMPFOLDER}/power.dc.good - fi - - else - [ -f ${PW_TMPFOLDER}/power.dc.good ] && rm -f ${PW_TMPFOLDER}/power.dc.good - [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_ABNORMAL}*60" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - LOW BATTERY Voltage" >> ${PW_LOGPOWER} - - if (( $(bc <<<"${BATTVOLT} <= ${min_battvolt_sys}") )); then - [ ! -f ${PW_TMPFOLDER}/power.dc.critlow ] && touch ${PW_TMPFOLDER}/power.dc.critlow - - if [ ! -f ${PW_TMPFOLDER}/shutdown.SYS.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.critlow)) -ge 30 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down all remaining servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "Battery Voltage REALLY LOW: ${BATTVOLT}'volts -Shutting down all remaining servers" 1 - SHUTDOWN_SYS POWER & - - fi - - elif (( $(bc <<<"${BATTVOLT} < ${min_battvolt}") )); then - [ ! -f ${PW_TMPFOLDER}/power.dc.low ] && touch ${PW_TMPFOLDER}/power.dc.low - [ -f ${PW_TMPFOLDER}/power.dc.critlow ] && rm -f ${PW_TMPFOLDER}/power.dc.critlow - - if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && (( $(bc <<<"${INPUTACV} < ${min_acvolt}") )) && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.dc.low)) -ge 30 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts -Shutting down main servers" 1 - SHUTDOWN_MAIN POWER & - - fi - - elif [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.MAINSITE.started ] && [ ! -f ${PW_TMPFOLDER}/shutdown.ALL.started ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_SHUTDOWN_MAINSERVERS_AFTER}*60" | bc) ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGFILE} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Shutting down main servers..." >> ${PW_LOGPOWER} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "Battery Voltage LOW: ${BATTVOLT}'volts -Shutting down main servers" 1 - SHUTDOWN_MAIN POWER & - fi - - fi - if [ -f ${PW_TMPFOLDER}/power.ac.low ] || [ -f ${PW_TMPFOLDER}/power.dc.low ]; then - volt_diff=`echo "scale=1; ${last_battvolt}-${BATTVOLT}" | bc` - (( $(bc <<<"${volt_diff} < 0") )) && volt_diff=`echo "scale=2; ${volt_diff} * -1" | bc` - if (( $(bc <<<"${volt_diff} >= 0.5") )); then - SENDNOTICE "BATTERY VOLTAGE CHANGE" "Battery Voltage: ${BATTVOLT}'volts" - echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${BATTVOLT}'volts - Battery Voltage Change" >> ${PW_LOGPOWER} - last_battvolt=${BATTVOLT} - fi - fi - - else - sleep 20s - fi - - end=`date +%s`; runtime=$((end-start)) - # echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s" - [ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s ###|| echo "Not sleeping, ran for ${runtime}s previsoulsy" - done # & -} - -OFFSITE_CHECKPOWER_SERVICE(){ - startup=true - runEvery=10 #SECONDS - while true; do - start=$(date +%s) - source ${PW_FOLDER}/defaults.inc - - PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ') - # TEST=true - # TEST=full - if [ "${TEST}" != "" ]; then - PW_OFFSITE_SHUTDOWNSERVERS_AT_BATTCAP=98 #percent - PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes - PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes - TESTMODE="### TEST MODE ACTIVE ### - - " - else - TESTMODE="" - fi - - # upsinfo=$(upsc CP1500PFCLCD) - upsinfo=$(upsc CP1500PFCLCD 2>&1) - if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then - rm -f ${PW_TMPFOLDER}/power.ups.losscomm - OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') - OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') - OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` - OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` - OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') - OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` - OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') - - if [ ! -d /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']} ]; then - /bin/dtoverlay w1-gpio gpiopin=4 - sleep 2s - fi - if [ -d /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']} ]; then - temp_c=$(cat /sys/bus/w1/devices/${PW_SENSOR_DEVICEID['Offsite-ServerRoom']}/w1_slave | tail -n1 | cut -d "=" -f2) - OFFSITE_TEMP=$(echo "scale=2; ${temp_c}/1000" | bc) - else - OFFSITE_TEMP=ERR - - fi - - SENDNOTICE2(){ - MESSAGE="${TESTMODE}${2} - -BATTERY CHARGE: ${OFFSITE_BATT_CAP}'% -BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'mins -AC VOLTAGE: ${OFFSITE_VOLTIN}'volts -WATT USAGE: ${OFFSITE_WATTS}'watts -AMBIENT TEMP: ${OFFSITE_TEMP}'F" - [ "${3}" != "" ] && PRIORITY=${3} || PRIORITY=0 - SENDNOTICE "${1}" "${MESSAGE}" ${PRIORITY} cosmic - } - if [ "${startup}" = "true" ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE} - OFFSITE_SENDPOWER "OFFSITE POWER Service Startup" & - startup=false - fi - - if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then - if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE} - SENDNOTICE2 "OFFSITE POWER" "Batteries are fully recharged" - rm -f ${PW_TMPFOLDER}/power.offsite.charging - # elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then - elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then - # touch ${PW_TMPFOLDER}/power.offsite.charging - echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging - fi - if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then - if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE} - SENDNOTICE2 "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" - fi - - rm -f ${PW_TMPFOLDER}/power.offsite.ac.* - fi - if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then - if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then - touch ${PW_TMPFOLDER}/power.offsite.restored - - elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE} - SENDNOTICE2 "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage" - for host in ${PW_OFFSITEHOSTS_IDRAC}; do - if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then - ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & - else - echo "TESTMODE: Powering up '${host}'" - fi - done - rm -f ${PW_TMPFOLDER}/power.offsite.restored - rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE - fi - fi - logtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) - if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER} - fi - else - [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low - - if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then - # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE} - SENDNOTICE2 "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1 - touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice - fi - - if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_BATTCAP} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE} - SENDNOTICE2 "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1 - touch ${PW_TMPFOLDER}/shutdown.OFFSITE - for host in ${PW_OFFSITEHOSTS}; do - if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then - ${PW_FOLDER}/esxi-scripts/offsite-host-shutdown.ps1 ${host} >/dev/null 2>&1 & - fi - done - fi - if [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER} - fi - fi - - else - if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then - touch ${PW_TMPFOLDER}/power.ups.losscomm - - elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE} - SENDNOTICE2 "NUT DRIVER FAILURE" "OMG WHAT DO WE DO, FIX ME!!! I used to be cool and say..... Restarting NUT service" - # /usr/local/etc/rc.d/nut.sh restart - # sleep 10s - touch ${PW_TMPFOLDER}/power.ups.losscomm - fi - fi - - end=`date +%s`; runtime=$((end-start)) - # echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s" - [ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s ###|| echo "Not sleeping, ran for ${runtime}s previsoulsy" - done -} - - -POWERLOGGER_SERVICE(){ - SENDPOWER "POWER LOGGING Service Startup" & - while true; do - LogDataTime=`date +'%Y-%m-%d %H:%M:%S'` - source ${PW_FOLDER}/defaults.inc - - INVSTATSINFO=$(curl -m 3 -s "http://${PW_INVERTER_IP}/stats.json") - if [ "${INVSTATSINFO}" != "" ]; then - INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'` - INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'` - OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'` - OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'` - OUTPUTW=`echo ${INVSTATSINFO} | jq '.outputs .outW'` - BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'` - BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'` - - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['inv-in']}','${LogDataTime}','${INPUTV}','${INPUTA}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['inv-out']}','${LogDataTime}','${OUTPUTV}','${OUTPUTA}','${OUTPUTW}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('${PW_POWERSENSOR_ID['battery']}','${LogDataTime}','${BATTV}','${BATTA}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - - QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${LogDataTime}','${INVSTATSINFO}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - - if [ "$(CHECK_HOST ${PW_UPS_IP})" != "false" ]; then - SVRRM_VOLT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.2.1.1 | sed 's/.*: //') - SVRRM_VOLT=`echo "scale=1; ${SVRRM_VOLT}/10" | bc` - SVRRM_WATT=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.4.1.1 | sed 's/.*: //') - SVRRM_AMP=$(snmpwalk -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.3.2.1.3.1.1 | sed 's/.*: //') - SVRRM_AMP=`echo "scale=2; ${SVRRM_AMP}/100" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total']}','${LogDataTime}','${SVRRM_VOLT}','${SVRRM_AMP}','${SVRRM_WATT}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - - if [ "$(CHECK_HOST ${PW_APCPDU_IP})" != "false" ]; then - APCPDU_AMP=$(snmpwalk -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.2.3.1.1.2.1 | sed 's/.*: //') - APCPDU_AMP=`echo "scale=1; ${APCPDU_AMP}/10" | bc` - if [ "${SVRRM_VOLT}" != "" ]; then - APCPDU_WATT=`echo "scale=2; ${APCPDU_AMP}*${SVRRM_VOLT}" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['server-rack']}','${LogDataTime}','${SVRRM_VOLT}','${APCPDU_AMP}','${APCPDU_WATT}')" - else - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`) VALUES ('12','${LogDataTime}','${APCPDU_AMP}')" - fi - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - - if [ "${SVRRM_VOLT}" != "" ] && [ "${APCPDU_AMP}" != "" ]; then - NETRK_AMP=`echo "scale=2; ${SVRRM_AMP}-${APCPDU_AMP}" | bc` - NETRK_WATT=$(ROUND_NUMBER `echo "scale=2; ${SVRRM_WATT}-${APCPDU_WATT}" | bc` 0) - - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['network-rack']}','${LogDataTime}','${SVRRM_VOLT}','${NETRK_AMP}','${NETRK_WATT}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - - AIRCON_INFO=$(curl -m 3 -s http://${PW_REMOTE_SENSORS['ServerRoom-AC']}:8888/status/${PW_SENSOR_DEVICEID['ServerRoom-AC']} | jq '.dps') - if [ "${AIRCON_INFO}" != "" ]; then - AIRCON_AMP=$(echo "${AIRCON_INFO}" | grep '"18":' | cut -d' ' -f 4 | sed 's/,//g') - AIRCON_AMP=$(ROUND_NUMBER $(echo "scale=3; ${AIRCON_AMP}/1000" | bc -l) 2) - AIRCON_WATT=$(echo "${AIRCON_INFO}" | grep '"19":' | cut -d' ' -f 4 | sed 's/,//g') - AIRCON_WATT=$(echo "scale=1; ${AIRCON_WATT}/10" | bc -l) - AIRCON_VOLT=$(echo "${AIRCON_INFO}" | grep '"20":' | cut -d' ' -f 4 | sed 's/,//g') - AIRCON_VOLT=$(echo "scale=3; ${AIRCON_VOLT}/10" | bc -l) - - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['ServerRoom-AC']}','${LogDataTime}','${AIRCON_VOLT}','${AIRCON_AMP}','${AIRCON_WATT}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - if [ "${SVRRM_WATT}" != "" ]; then - SVRRM_AMP_WAC=$(echo "scale=2; ${SVRRM_AMP}+${AIRCON_AMP}" | bc -l) - SVRRM_WATT_WAC=$(echo "scale=2; ${SVRRM_WATT}+${AIRCON_WATT}" | bc -l) - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['total-wac']}','${LogDataTime}','${SVRRM_AMP_WAC}','${SVRRM_WATT_WAC}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - fi - - # upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') - upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_OFFSITE_MONITOR} 'upsc CP1500PFCLCD' 2>&1) - if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then - OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //') - OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') - OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` - OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` - OFFSITE_AMPS=`echo "scale=2; ${OFFSITE_WATTS}/${OFFSITE_VOLTOUT}" | bc` - - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID['offsite-ups']}','${LogDataTime}','${OFFSITE_VOLTOUT}','${OFFSITE_AMPS}','${OFFSITE_WATTS}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - - for SENSOR in ${PW_SENSOR_ORD[@]}; do - if [ "${PW_HOST_IDRACS[${SENSOR}]}" != "" ] && [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSOR}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then - HOST_WATT_SENSORS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1 -O vq | sed -e 's/"//g') - if [[ "${HOST_WATTS}" != *"Timeout"* ]] && [ "${HOST_WATT_SENSORS}" != "" ]; then - IFS=$'\n' - read -rd '' -a HOST_WATT_SENSORS <<<"${HOST_WATT_SENSORS}" - unset IFS - [ -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.disable - [ -f ${PW_TMPFOLDER}/shutdown.${PW_HOST_ADDRESSES[${SENSOR}]} ] && rm -f ${PW_TMPFOLDER}/shutdown.${PW_HOST_ADDRESSES[${SENSOR}]} - [ -f ${PW_TMPFOLDER}/shutdown.ALL ] && rm -f ${PW_TMPFOLDER}/shutdown.ALL - [[ "${PW_OFFSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE - [[ "${PW_MAINSITE_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.MAINSITE - [[ "${PW_MAIN_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.MAIN - [[ "${PW_SYS_HOSTS}" = *"${PW_HOST_ADDRESSES[${SENSOR}]}"* ]] && rm -f ${PW_TMPFOLDER}/shutdown.SYS - - - PS1='0.0'; PS2='0.0'; HOST_WATTS=0 - for s in {1..5}; do - DP=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.8.1.${s} | sed 's/.*: //') - if [[ "${DP}" = *"System Board"* ]]; then - found=1 - HOST_WATTS=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - - elif [[ "${DP}" = *"PS1"* ]]; then - PS1=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - elif [[ "${DP}" = *"PS2"* ]]; then - PS2=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_HOST_IDRACS[${SENSOR}]} iso.3.6.1.4.1.674.10892.5.4.600.30.1.6.1.${s} | sed 's/.*: //') - fi - done - if [ ${found} -eq 1 ] && [ "${HOST_WATTS}" != "0" ] && [ "${HOST_WATTS}" != "" ] && [[ "${HOST_WATTS}" != *"Timeout"* ]]; then - [ "${PS1}" != "0.0" ] && PS1=`echo "scale=1; ${PS1}/10" | bc` && (( $(bc <<<"${PS1} < 1") )) && [[ "${PS1}" = *"."* ]] && PS1="0${PS1}" - [ "${PS2}" != "0.0" ] && PS2=`echo "scale=1; ${PS2}/10" | bc` && (( $(bc <<<"${PS2} < 1") )) && [[ "${PS2}" = *"."* ]] && PS2="0${PS2}" - HOST_AMPS=`echo "scale=1; ${PS1}+${PS2}" | bc` - QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`amp\`, \`watt\`) VALUES ('${PW_POWERSENSOR_ID[${SENSOR}]}','${LogDataTime}','${HOST_AMPS}','${HOST_WATTS}')" - ${mysql_conn} -e "USE servermonitor; ${QRY}" - fi - fi - fi - done - - if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*.low")" != "" ]; then - sleep ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}m - - else - sleep ${PW_MYSQL_LOG_INTERVAL_NORMAL}m - fi - - done -} - -HOSTPOWER(){ - if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then - idracip=${1} - elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then - tmp=${PW_ESXI_HOST_NAMES[${1}]} - idracip=${PW_HOST_IDRACS[${tmp}]} - elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then - idracip=${PW_HOST_IDRACS[iDS-vMS-Host${1}]} - elif ([ "${1::1}" = "o" ] || [ "${1::1}" = "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${1: -1}]}" != "" ]; then - idracip=${PW_HOST_IDRACS[OFF-vMS-Host${1: -1}]} - elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then - idracip=${PW_HOST_IDRACS[${1}]} - else - idracip='' - fi - - - echo -e "\n${idsCL[Yellow]}POWERING ${2^^} '${PW_iDRACHOST_SHORTNAMES[${idracip}]}'" - DIVIDER . yellow ${PRI_WIDTH} - - if [ "${idracip}" != "" ]; then - - if [ "${2^^}" = "ON" ] || [ "${2^^}" = "OFF" ] || [ "${2^^}" = "STATUS" ]; then - if [ "${idracip}" = "10.10.2.20" ] && [ "${2^^}" = "ON" ]; then - MSG="Powering ON the SAS Enclosure" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "\n${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - current_status=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 | sed 's/.*: //') - if [ "${current_status}" != "" ] && [ "${current_status}" != "1" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 1 >/dev/null 2>&1) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - elif [ "${current_status}" = "1" ]; then - echo -e "${idsCL[Green]}SAS Enclosure Already Powered ON${idsCL[Default]}\n" - fi - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - fi - - if [ "${TEST}" != "true" ]; then - current_status=$(ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep "Power is ${2,,}") - if [ "${current_status}" = "" ]; then - echo -e "${idsCL[Green]}" - ipmitool -I lanplus -H ${idracip} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power ${2,,} - else - echo -e "\n${idsCL[LightCyan]}Host is already Powered ${2^^}${idsCL[Default]}" - - fi - else - echo -e "\n${idsCL[Yellow]}(TEST MODE)" - fi - echo -e "${idsCL[Default]}" - - if [ "${idracip}" = "10.10.2.20" ] && [ "${2^^}" = "OFF" ]; then - MSG="Powering OFF the SAS Enclosure" - c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done - echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: " - if [ "${TEST}" != "true" ]; then - if [ "$(CHECK_HOST 10.10.2.10)" = "false" ]; then - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1) - sleep 3s - echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" - else - echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n" - fi - else - echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" - fi - fi - else - echo -e "${idsCL[LightRed]}'${2^^}' command is not recognized${idsCL[Default]}" - fi - else - echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}" - fi - echo -e "${idsCL[Default]}\n" -} - -PDUCONTROL(){ - function exitspacing { - echo -e "\n\033[K\n\033[K" - exit 0 - } - trap exitspacing EXIT - - while [ $# -gt 0 ]; do - case "$1" in - -port) PORT_SEL=${2};; - esac - shift 1 - done - echo - DIVIDER . yellow ${PRI_WIDTH} - echo -e "${idsCL[Yellow]}APC-PDU Power Control" - DIVIDER true yellow ${PRI_WIDTH} - until [ "${PORTSEL^^}" = "E" ]; do - [ "${PORT_SEL}" != "" ] && PORTSEL=${PORT_SEL} - if [ "${PORTSEL}" = "" ]; then - PORT_DESC=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2 -O vq 2> /dev/null | sed -e 's/"//g') - PORT_POWER=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4 -O vq 2> /dev/null | sed -e 's/"//g') - IFS=$'\n' - read -rd '' -a PORT_DESC <<<"${PORT_DESC}" - read -rd '' -a PORT_POWER <<<"${PORT_POWER}" - unset IFS - for p in "${!PORT_DESC[@]}"; do - port=$(expr ${p} + 1) - [ ${PORT_POWER[${p}]} -eq 1 ] && portpower_disp="${idsCL[Green]}ON" || portpower_disp="${idsCL[LightRed]}OFF" - [ ${port} -lt 10 ] && pp=" ${port}" || pp=${port} - c=0; spc=''; spc1=`expr 28 - ${#PORT_DESC[${p}]}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done - echo -e "${pp}) ${idsCL[LightCyan]}${PORT_DESC[${p}]}${spc}${idsCL[Default]}: ${portpower_disp}${idsCL[Default]}" - done - echo -e "" - echo -e "" - echo -e "(${idsCL[Yellow]}E${idsCL[Default]})xit" - echo -e "" - echo -e "\033[5A" - # until [ "${PORTSEL}" != "" ]; do - echo -en "\n${idsCL[LightCyan]}Enter the port number to control: ${idsCL[Default]}" - read PORTSEL - echo - # done - fi - - [ "${PORTSEL^^}" = "E" ] && exit 0 - - [ "${PORTSEL}" != "" ] && tp=$(expr ${PORTSEL} - 1) - if [ "${PORTSEL}" != "" ] && [[ "${PORTSEL}" =~ ^[0-9]+$ ]] && [ "${PORT_DESC[${tp}]}" != "" ] && [ ${PORTSEL} -ne 0 ]; then - # portsel_power=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g') - [ ${PORT_POWER[${tp}]} -eq 1 ] && portsel_power_disp="${idsCL[Green]}ON" || portsel_power_disp="${idsCL[LightRed]}OFF" - [ ${PORT_POWER[${tp}]} -eq 2 ] && portsel_change_power_disp="${idsCL[Green]}ON" || portsel_change_power_disp="${idsCL[LightRed]}OFF" - - # portdesc=$(snmpwalk -t 3 -r 0 -v 1 -c public ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.2.${PORTSEL} -O vq 2> /dev/null | sed -e 's/"//g') - DIVIDER . yellow 60 - echo -e "${idsCL[Yellow]}Port ${idsCL[LightYellow]}${PORTSEL}${idsCL[Default]}) ${idsCL[LightCyan]}${PORT_DESC[${tp}]} ${idsCL[Default]}: ${portsel_power_disp}${idsCL[Default]}\n" - - echo -en "${idsCL[LightCyan]}Are you sure you wish to turn ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} (y/N): " - read -n 1 portchange_confirm - case "${portchange_confirm}" in - [Yy]) - [ ${PORT_POWER[${tp}]} -eq 1 ] && portsel_change=2 || portsel_change=1 - echo -en "\n\n${idsCL[LightCyan]}Turning ${portsel_change_power_disp}${idsCL[LightCyan]} port ${idsCL[Yellow]}${PORTSEL}${idsCL[Default]} ... " - run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.${PORTSEL} i ${portsel_change} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - echo - ;; - - esac - [ "${PORT_SEL}" = "" ] && unset PORTSEL - else - echo -e "\n${idsCL[LightRed]}Not a valid port number${idsCL[Default]}" - unset PORTSEL - - fi - [ "${PORT_SEL}" != "" ] && exit 0 || echo - done - -} - -SENDPOWER(){ - st_power=`${PW_SCRIPT} check power -m -p -r` - SENDNOTICE "${1}" "${st_power}" -} - -OFFSITE_SENDPOWER(){ - st_power=`${PW_SCRIPT} check power -o -p -r` - SENDNOTICE "${1}" "${st_power}" 0 cosmic -} - - diff --git a/offsite-power-check.sh b/offsite-power-check.sh deleted file mode 100755 index 2b73d225..00000000 --- a/offsite-power-check.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env sh - -PW_FOLDER='/opt/idssys/powerwall' -PW_LOGFOLDER=${PW_FOLDER}/logs -PW_LOGARCHIVE=${PW_FOLDER}/logs/archive -PW_TMPFOLDER=${PW_FOLDER}/.tmp -[ ! -d ${PW_LOGFOLDER} ] && mkdir ${PW_LOGFOLDER} -[ ! -d ${PW_LOGARCHIVE} ] && mkdir ${PW_LOGARCHIVE} -[ ! -d ${PW_TMPFOLDER} ] && mkdir ${PW_TMPFOLDER} -PW_LOGFILE=${PW_LOGFOLDER}/logfile -PW_LOGPOWER=${PW_LOGFOLDER}/log-power -[ ! -f ${PW_LOGFILE} ] && touch ${PW_LOGFILE} -[ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER} - -PWO_START(){ - startup=true - while true; do - - . /opt/idssys/powerwall/conf/settings.conf - PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ') - # TEST=true - # TEST=full - if [ "${TEST}" != "" ]; then - PW_OFFSITE_SHUTDOWNSERVERS_AT_BATTCAP=98 #percent - PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes - PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes - TESTMODE="### TEST MODE ACTIVE ### - - " - else - TESTMODE="" - fi - - # upsinfo=$(upsc CP1500PFCLCD) - upsinfo=$(upsc CP1500PFCLCD 2>&1) - if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then - rm -f ${PW_TMPFOLDER}/power.ups.losscomm - OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') - OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') - OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` - OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` - OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') - OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` - OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') - SENDNOTICE(){ - MESSAGE="${TESTMODE}${2} - -BATTERY CHARGE: ${OFFSITE_BATT_CAP}'% -BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'mins -AC VOLTAGE: ${OFFSITE_VOLTIN}'volts -WATT USAGE: ${OFFSITE_WATTS}'watts - -$(date)" - if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi - curl -m 3 -sd "token=${PW_PUSHOVER_APP_TOKEN}&user=${PW_PUSHOVER_USER_TOKEN}&message=$(echo "${MESSAGE}" | sed -e "s/%/%25/g")&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1 - } - if [ "${startup}" = "true" ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE POWER MONITOR" "Service starting" - startup=false - fi - - if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then - if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE POWER" "Batteries are fully recharged" - rm -f ${PW_TMPFOLDER}/power.offsite.charging - # elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then - elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then - # touch ${PW_TMPFOLDER}/power.offsite.charging - echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging - fi - if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then - if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" - fi - - rm -f ${PW_TMPFOLDER}/power.offsite.ac.* - fi - if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then - if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then - touch ${PW_TMPFOLDER}/power.offsite.restored - - elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage" - for host in ${PW_OFFSITEHOSTS_IDRAC}; do - if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then - ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - else - echo "TESTMODE: Powering up '${host}'" - fi - done - rm -f ${PW_TMPFOLDER}/power.offsite.restored - rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE - fi - fi - logtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) - if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER} - fi - else - [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low - - if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then - # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1 - touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice - fi - - if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_BATTCAP} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE} - SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1 - touch ${PW_TMPFOLDER}/shutdown.OFFSITE - for host in ${PW_OFFSITEHOSTS}; do - if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & - fi - done - fi - if [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER} - fi - fi - - else - if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then - touch ${PW_TMPFOLDER}/power.ups.losscomm - - elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then - echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE} - SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service" - /usr/local/etc/rc.d/nut.sh restart - sleep 10s - touch ${PW_TMPFOLDER}/power.ups.losscomm - fi - fi - sleep 5s - done -} - -PWO_STOP(){ - ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}' | xargs kill -} - -PWO_RESTART() { - PWO_STOP - sleep 2s - /opt/idssys/powerwall/offsite-power-check.sh start & - -} - -LOGROTATE(){ - lm=`expr $(date +'%m') - 1` - if [ ${lm} = 0 ]; then - lm=12 - ly=`expr $(date +'%Y') - 1` - else - ly=$(date +'%Y') - fi - [ ${lm} -lt 10 ] && lm="0${lm}" - ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm} - mkdir -p ${ARCHIVEFOLDER} - PWO_STOP - mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null - PWO_START - i=0 - for folder in $(ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort -r); do - i=`expr ${i} + 1` - [ ${i} -gt 12 ] && rm -Rf ${folder} - done -} - -case $1 in - start) PWO_START &;; - stop) PWO_STOP;; - restart) PWO_RESTART;; - logrotate) LOGROTATE;; - update) - if curl -m 5 -s --head --request GET https://git.schroedercity.com | grep "HTTP/2 200" > /dev/null; then - # cd /opt/idssys/defaults - # if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then - # git fetch origin master >/dev/null 2>&1 - # git reset --hard origin/master >/dev/null 2>&1 - # git reflog expire --expire=now --all >/dev/null 2>&1 - # git repack -ad >/dev/null 2>&1 - # git prune >/dev/null 2>&1 - # git pull >/dev/null 2>&1 - # git submodule update --remote >/dev/null 2>&1 - # fi - cd ${PW_FOLDER} - if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then - git fetch origin master >/dev/null 2>&1 - git reset --hard origin/master >/dev/null 2>&1 - git reflog expire --expire=now --all >/dev/null 2>&1 - git repack -ad >/dev/null 2>&1 - git prune >/dev/null 2>&1 - git pull >/dev/null 2>&1 - echo "Updated, restarting services" - /opt/idssys/powerwall/offsite-power-check.sh restart & - else - echo "No update available" - fi - else - echo "Cannot connect to GIT server" - fi - ;; -esac - - -exit 0 - - - - - -