update
This commit is contained in:
130
inc/power.inc.sh
130
inc/power.inc.sh
@@ -554,6 +554,136 @@ Shutting down main servers" 1
|
||||
done # &
|
||||
}
|
||||
|
||||
OFFSITE_CHECKPOWER_SERVICE(){
|
||||
startup=true
|
||||
runEvery=10 #SECONDS
|
||||
while true; do
|
||||
start=$(date +%s)
|
||||
source ${PW_FOLDER}/defaults.inc
|
||||
|
||||
PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ')
|
||||
# TEST=true
|
||||
# TEST=full
|
||||
if [ "${TEST}" != "" ]; then
|
||||
PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD=95 #percent
|
||||
PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes
|
||||
PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes
|
||||
TESTMODE="### TEST MODE ACTIVE ###
|
||||
|
||||
"
|
||||
else
|
||||
TESTMODE=""
|
||||
fi
|
||||
|
||||
# upsinfo=$(upsc CP1500PFCLCD)
|
||||
upsinfo=$(upsc offsite-ups 2>&1)
|
||||
if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then
|
||||
rm -f ${PW_TMPFOLDER}/power.ups.losscomm
|
||||
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
|
||||
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
|
||||
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
|
||||
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
|
||||
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
|
||||
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
|
||||
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
|
||||
|
||||
if [ "${startup}" = "true" ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE POWER MONITOR" "Service starting"
|
||||
startup=false
|
||||
fi
|
||||
|
||||
if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then
|
||||
if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE POWER" "Batteries are fully recharged"
|
||||
rm -f ${PW_TMPFOLDER}/power.offsite.charging
|
||||
# elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then
|
||||
elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then
|
||||
# touch ${PW_TMPFOLDER}/power.offsite.charging
|
||||
echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging
|
||||
fi
|
||||
if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
|
||||
if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
|
||||
fi
|
||||
|
||||
rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
|
||||
fi
|
||||
if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
|
||||
if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
|
||||
touch ${PW_TMPFOLDER}/power.offsite.restored
|
||||
|
||||
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage"
|
||||
for host in ${PW_OFFSITEHOSTS_IDRAC}; do
|
||||
if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then
|
||||
ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
else
|
||||
echo "TESTMODE: Powering up '${host}'"
|
||||
fi
|
||||
done
|
||||
rm -f ${PW_TMPFOLDER}/power.offsite.restored
|
||||
rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE
|
||||
fi
|
||||
fi
|
||||
logtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
|
||||
if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER}
|
||||
fi
|
||||
else
|
||||
[ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low
|
||||
|
||||
if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then
|
||||
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1
|
||||
touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice
|
||||
fi
|
||||
|
||||
if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
|
||||
touch ${PW_TMPFOLDER}/shutdown.OFFSITE
|
||||
for host in ${PW_OFFSITEHOSTS}; do
|
||||
if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
# run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
|
||||
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh &
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER}
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then
|
||||
touch ${PW_TMPFOLDER}/power.ups.losscomm
|
||||
|
||||
elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE}
|
||||
SENDNOTICE "NUT DRIVER FAILURE" "OMG WHAT DO WE DO, FIX ME!!! I used to be cool and say..... Restarting NUT service"
|
||||
# /usr/local/etc/rc.d/nut.sh restart
|
||||
# sleep 10s
|
||||
touch ${PW_TMPFOLDER}/power.ups.losscomm
|
||||
fi
|
||||
fi
|
||||
|
||||
end=`date +%s`; runtime=$((end-start))
|
||||
echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s"
|
||||
[ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s || echo "Not sleeping, ran for ${runtime}s previsoulsy"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
POWERLOGGER_SERVICE(){
|
||||
SENDPOWER "POWER LOGGING Service Startup" &
|
||||
while true; do
|
||||
|
||||
@@ -865,6 +865,8 @@ fi
|
||||
CHECKTEMP_SERVICE
|
||||
elif [ "${2}" == "power" ]; then
|
||||
CHECKPOWER_SERVICE
|
||||
elif [ "${2}" == "offsitepower" ]; then
|
||||
OFFSITE_CHECKPOWER_SERVICE
|
||||
elif [ "${2}" == "powerlogger" ]; then
|
||||
POWERLOGGER_SERVICE
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user