powerwall/offsite-power-check.sh

#!/usr/bin/env sh

PW_FOLDER='/opt/idssys/powerwall'
PW_LOGFOLDER=${PW_FOLDER}/logs
PW_LOGFILE=${PW_LOGFOLDER}/logfile
PW_LOGPOWER=${PW_LOGFOLDER}/log-power
PW_TMPFOLDER=${PW_FOLDER}/.tmp
[ ! -f ${PW_LOGFILE} ] && touch ${PW_LOGFILE}
[ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER}

. /opt/idssys/powerwall/settings.conf

SHUTDOWNSERVERS_AT_LOAD=80	#percent
WAIT_TO_POWER_SERVERS=5		#minutes
LOG_INTERVAL_NORMAL=60		#minutes
LOG_INTERVAL_ABNORMAL=30	#seconds

# TEST=true  #full
if [ "$TEST" != "" ]; then
	SHUTDOWNSERVERS_AT_LOAD=95
	WAIT_TO_POWER_SERVERS=1
	LOG_INTERVAL_NORMAL=1
	LOG_INTERVAL_ABNORMAL=1
	TESTMODE="### TEST MODE ACTIVE ###
	
	"
else
	TESTMODE=""
fi

PWO_START(){
	startup=true
	while true; do
		upsinfo=$(upsc CP1500PFCLCD)
		if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then
			OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
			OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
			OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
			OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
			SENDNOTICE(){
				MESSAGE="${TESTMODE}${2}

BATTERY CHARGE: ${OFFSITE_BATT_CAP}%25
BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'Mins
AC VOLTAGE: ${OFFSITE_VOLTIN}'Volts

$(date)"
				if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi
				curl -sd "token=${PUSHOVER_APP_TOKEN}&user=${PUSHOVER_USER_TOKEN}&message=${MESSAGE}&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1
			}
			if [ "${startup}" == "true" ]; then
				echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Service startup" >> ${PW_LOGFILE}
				SENDNOTICE "OFFSITE POWER MONITOR" "Service starting"
				startup=false
			fi
			
			if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then
				if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
					if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
						echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Offsite Power Restored" >> ${PW_LOGFILE}
						SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${WAIT_TO_POWER_SERVERS}'Mins before turning host(s) back on"
					fi
					
					rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
				fi
				if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
					if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
						touch ${PW_TMPFOLDER}/power.offsite.restored

					elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
						echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE}
						SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage"
						([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ipmitool -I lanplus -H 10.2.1.21 -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
						rm -f ${PW_TMPFOLDER}/power.offsite.*
					fi
				fi
				logtime=$(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER}))
				if [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_ABNORMAL}" | bc) ]); then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Normal" >> ${PW_LOGPOWER}
				fi
			else
				if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low)) -gt 240 ]; then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE}
					SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1
					touch ${PW_TMPFOLDER}/power.offsite.ac.low
				fi

				if [ ${OFFSITE_BATT_CAP%.*} -lt ${SHUTDOWNSERVERS_AT_LOAD} ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
					echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Powering Down Hosts!" >> ${PW_LOGFILE}
					SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
					touch ${PW_TMPFOLDER}/power.offsite.off
					([ "$TEST" = "" ] || [ "$TEST" = "full" ]) && ssh root@10.2.1.11 /vmfs/volumes/652c94ef-ee49895b-365d-00109b3fea40/.SCRIPTS/esxi-shutdown2.sh &
				fi
				[ $(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) -gt $(echo "scale=2; ${LOG_INTERVAL_ABNORMAL}" | bc) ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - OFF/LOW POWER!!" >> ${PW_LOGPOWER}
			fi
		
		else
			echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'Volts - ${OFFSITE_BATT_CAP}% Battery - Restarting NUT service" >> ${PW_LOGFILE}
			SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service"
			/usr/local/etc/rc.d/nut.sh restart
			sleep 10s
		fi
		sleep 5s
	done
}

PWO_STOP(){
	ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}' | xargs kill
}


PWO_RESTART() {
	PWO_STOP
	sleep 2s
	/opt/idssys/powerwall/offsite-power-check.sh start &

}

case $1 in
	start) PWO_START &;;
	stop) PWO_STOP;;
	restart) PWO_RESTART;;
	update)
		cd ${PW_FOLDER}
		if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
			git fetch origin master >/dev/null 2>&1
			git reset --hard origin/master >/dev/null 2>&1
			git reflog expire --expire=now --all >/dev/null 2>&1
			git repack -ad >/dev/null 2>&1
			git prune >/dev/null 2>&1
			git pull >/dev/null 2>&1
			# /bin/chmod +x /opt/idssys/powerwall/offsite-power-check.sh 2>&1
			/opt/idssys/powerwall/offsite-power-check.sh restart &
		fi
	;;
esac


exit 0