#!/usr/bin/env sh PW_FOLDER='/opt/idssys/powerwall' PW_LOGFOLDER=${PW_FOLDER}/logs PW_LOGARCHIVE=${PW_FOLDER}/logs/archive PW_TMPFOLDER=${PW_FOLDER}/.tmp [ ! -d ${PW_LOGFOLDER} ] && mkdir ${PW_LOGFOLDER} [ ! -d ${PW_LOGARCHIVE} ] && mkdir ${PW_LOGARCHIVE} [ ! -d ${PW_TMPFOLDER} ] && mkdir ${PW_TMPFOLDER} PW_LOGFILE=${PW_LOGFOLDER}/logfile PW_LOGPOWER=${PW_LOGFOLDER}/log-power [ ! -f ${PW_LOGFILE} ] && touch ${PW_LOGFILE} [ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER} PWO_START(){ startup=true while true; do . /opt/idssys/powerwall/settings.conf PW_OFFSITEHOSTS=$(echo ${PW_OFFSITE_HOSTS} | tr ',' ' ') # TEST=true # TEST=full if [ "${TEST}" != "" ]; then PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD=95 #percent PW_OFFSITE_SHUTDOWNSERVERS_AFTER=1 #minutes PW_OFFSITE_WAIT_TO_POWER_SERVERS=1 #minutes TESTMODE="### TEST MODE ACTIVE ### " else TESTMODE="" fi # upsinfo=$(upsc CP1500PFCLCD) upsinfo=$(upsc offsite-ups) if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then rm -f ${PW_TMPFOLDER}/power.ups.losscomm OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc` OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc` OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') SENDNOTICE(){ MESSAGE="${TESTMODE}${2} BATTERY CHARGE: ${OFFSITE_BATT_CAP}'% BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'mins AC VOLTAGE: ${OFFSITE_VOLTIN}'volts WATT USAGE: ${OFFSITE_WATTS}'watts $(date)" if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi curl -m 3 -sd "token=${PW_PUSHOVER_APP_TOKEN}&user=${PW_PUSHOVER_USER_TOKEN}&message=$(echo "${MESSAGE}" | sed -e "s/%/%25/g")&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1 } if [ "${startup}" == "true" ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Service startup" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE POWER MONITOR" "Service starting" startup=false fi if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Charging Complete" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE POWER" "Batteries are fully recharged" rm -f ${PW_TMPFOLDER}/power.offsite.charging # elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && ([ ! -f ${PW_TMPFOLDER}/power.offsite.charging ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.charging)) -gt 60 ]); then # touch ${PW_TMPFOLDER}/power.offsite.charging echo "${OFFSITE_BATT_CAP%.*}%" >| ${PW_TMPFOLDER}/power.offsite.charging fi if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Offsite Power Restored" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on" fi rm -f ${PW_TMPFOLDER}/power.offsite.ac.* fi if [ -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then touch ${PW_TMPFOLDER}/power.offsite.restored elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${PW_OFFSITE_WAIT_TO_POWER_SERVERS}*60" | bc) ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage" for host in ${PW_OFFSITEHOSTS_IDRAC}; do if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) else echo "TESTMODE: Powering up '${host}'" fi done rm -f ${PW_TMPFOLDER}/power.offsite.restored rm -f ${PW_TMPFOLDER}/shutdown.OFFSITE fi fi logtime=$(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) if [ ${logtime} -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${PW_LOG_INTERVAL_CHARGING} ]); then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER} fi else [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt $(echo "scale=2; ${PW_RENOTIFY_LOWPOWER}*60" | bc) ]; then # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1 touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice fi if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ] && [ ${OFFSITE_WATTS%.*} -gt 100 ] && ([ ${OFFSITE_BATT_CAP%.*} -le ${PW_OFFSITE_SHUTDOWNSERVERS_AT_LOAD} ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${PW_OFFSITE_SHUTDOWNSERVERS_AFTER}*60" | bc) ]); then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}'% Battery - Powering Down Hosts!" >> ${PW_LOGFILE} SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1 touch ${PW_TMPFOLDER}/shutdown.OFFSITE for host in ${PW_OFFSITEHOSTS}; do if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & fi done fi if [ $(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) -gt ${PW_LOG_INTERVAL_ABNORMAL} ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}'% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER} fi fi else if [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then touch ${PW_TMPFOLDER}/power.ups.losscomm elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 120 ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - 0.0'volts - 0'% Battery - Restarting NUT service" >> ${PW_LOGFILE} SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service" /usr/local/etc/rc.d/nut.sh restart sleep 10s touch ${PW_TMPFOLDER}/power.ups.losscomm fi fi sleep 5s done } PWO_STOP(){ ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}' | xargs kill } PWO_RESTART() { PWO_STOP sleep 2s /opt/idssys/powerwall/offsite-power-check.sh start & } LOGROTATE(){ lm=`expr $(date +'%m') - 1` if [ ${lm} = 0 ]; then lm=12 ly=`expr $(date +'%Y') - 1` else ly=$(date +'%Y') fi [ ${lm} -lt 10 ] && lm="0${lm}" ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm} mkdir -p ${ARCHIVEFOLDER} PWO_STOP mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null PWO_START i=0 for folder in $(ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort -r); do i=`expr ${i} + 1` [ ${i} -gt 12 ] && rm -Rf ${folder} done } case $1 in start) PWO_START;; stop) PWO_STOP;; restart) PWO_RESTART;; logrotate) LOGROTATE;; update) if curl -m 5 -s --head --request GET https://git.schroedercity.com | grep "HTTP/2 200" > /dev/null; then # cd /opt/idssys/defaults # if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then # git fetch origin master >/dev/null 2>&1 # git reset --hard origin/master >/dev/null 2>&1 # git reflog expire --expire=now --all >/dev/null 2>&1 # git repack -ad >/dev/null 2>&1 # git prune >/dev/null 2>&1 # git pull >/dev/null 2>&1 # git submodule update --remote >/dev/null 2>&1 # fi cd ${PW_FOLDER} if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then git fetch origin master >/dev/null 2>&1 git reset --hard origin/master >/dev/null 2>&1 git reflog expire --expire=now --all >/dev/null 2>&1 git repack -ad >/dev/null 2>&1 git prune >/dev/null 2>&1 git pull >/dev/null 2>&1 echo "Updated, restarting services" /opt/idssys/powerwall/offsite-power-check.sh restart & else echo "No update available" fi else echo "Cannot connect to GIT server" fi ;; esac exit 0