diff --git a/inc/power.inc.sh b/inc/power.inc.sh index 168c1e85..e981fee6 100644 --- a/inc/power.inc.sh +++ b/inc/power.inc.sh @@ -554,6 +554,136 @@ Shutting down main servers" 1 done # & } +OFFSITE_CHECKPOWER_SERVICE(){ + startup=true + runEvery=10 #SECONDS + while true; do + start=$(date +%s) + source ${PW_FOLDER}/defaults.inc + + PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ') + # TEST=true + # TEST=full + if [ "${TEST}" != "" ]; then + PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD=95 #percent + PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes + PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes + TESTMODE="### TEST MODE ACTIVE ### + + " + else + TESTMODE="" + fi + + # upsinfo=$(upsc CP1500PFCLCD) + upsinfo=$(upsc offsite-ups 2>&1) + if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then + rm -f ${PW_TMPFOLDER}/power.ups.losscomm + OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') + OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') + OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` + OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` + OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') + OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` + OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') + + if [ "${startup}" = "true" ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE POWER MONITOR" "Service starting" + startup=false + fi + + if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then + if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE POWER" "Batteries are fully recharged" + rm -f ${PW_TMPFOLDER}/power.offsite.charging + # elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then + elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then + # touch ${PW_TMPFOLDER}/power.offsite.charging + echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging + fi + if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then + if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" + fi + + rm -f ${PW_TMPFOLDER}/power.offsite.ac.* + fi + if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then + if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then + touch ${PW_TMPFOLDER}/power.offsite.restored + + elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage" + for host in ${PW_OFFSITEHOSTS_IDRAC}; do + if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then + ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + else + echo "TESTMODE: Powering up '${host}'" + fi + done + rm -f ${PW_TMPFOLDER}/power.offsite.restored + rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE + fi + fi + logtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) + if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER} + fi + else + [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low + + if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then + # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1 + touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice + fi + + if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE} + SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1 + touch ${PW_TMPFOLDER}/shutdown.OFFSITE + for host in ${PW_OFFSITEHOSTS}; do + if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & + fi + done + fi + if [ $(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER} + fi + fi + + else + if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then + touch ${PW_TMPFOLDER}/power.ups.losscomm + + elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then + echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE} + SENDNOTICE "NUT DRIVER FAILURE" "OMG WHAT DO WE DO, FIX ME!!! I used to be cool and say..... Restarting NUT service" + # /usr/local/etc/rc.d/nut.sh restart + # sleep 10s + touch ${PW_TMPFOLDER}/power.ups.losscomm + fi + fi + + end=`date +%s`; runtime=$((end-start)) + echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s" + [ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s || echo "Not sleeping, ran for ${runtime}s previsoulsy" + done +} + + POWERLOGGER_SERVICE(){ SENDPOWER "POWER LOGGING Service Startup" & while true; do diff --git a/powerwall.sh b/powerwall.sh index d3d6faa3..2b695002 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -865,6 +865,8 @@ fi CHECKTEMP_SERVICE elif [ "${2}" == "power" ]; then CHECKPOWER_SERVICE + elif [ "${2}" == "offsitepower" ]; then + OFFSITE_CHECKPOWER_SERVICE elif [ "${2}" == "powerlogger" ]; then POWERLOGGER_SERVICE else