219 lines
8.7 KiB
Bash
Executable File
219 lines
8.7 KiB
Bash
Executable File
#!/usr/bin/env sh
|
|
|
|
PW_FOLDER='/opt/idssys/powerwall'
|
|
PW_LOGFOLDER=${PW_FOLDER}/logs
|
|
PW_LOGARCHIVE=${PW_FOLDER}/logs/archive
|
|
PW_TMPFOLDER=${PW_FOLDER}/.tmp
|
|
[ ! -d ${PW_LOGFOLDER} ] && mkdir ${PW_LOGFOLDER}
|
|
[ ! -d ${PW_LOGARCHIVE} ] && mkdir ${PW_LOGARCHIVE}
|
|
[ ! -d ${PW_TMPFOLDER} ] && mkdir ${PW_TMPFOLDER}
|
|
PW_LOGFILE=${PW_LOGFOLDER}/logfile
|
|
PW_LOGPOWER=${PW_LOGFOLDER}/log-power
|
|
[ ! -f ${PW_LOGFILE} ] && touch ${PW_LOGFILE}
|
|
[ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER}
|
|
|
|
. /opt/idssys/powerwall/settings.conf
|
|
|
|
SHUTDOWNSERVERS_AT_LOAD=80 #percent
|
|
SHUTDOWNSERVERS_AFTER=5 #minutes
|
|
WAIT_TO_POWER_SERVERS=5 #minutes
|
|
LOG_INTERVAL_NORMAL=60 #minutes
|
|
LOG_INTERVAL_ABNORMAL=30 #seconds
|
|
LOG_INTERVAL_CHARGING=120 #seconds
|
|
RENOTIFY_LOWPOWER=10 #minutes
|
|
|
|
PW_OFFSITEHOSTS="10.2.1.11"
|
|
# PW_OFFSITEHOSTS="10.2.1.11 10.2.1.12"
|
|
|
|
# TEST=true #full
|
|
if [ "$TEST" != "" ]; then
|
|
SHUTDOWNSERVERS_AT_LOAD=70
|
|
WAIT_TO_POWER_SERVERS=1
|
|
LOG_INTERVAL_NORMAL=1
|
|
LOG_INTERVAL_ABNORMAL=1
|
|
TESTMODE="### TEST MODE ACTIVE ###
|
|
|
|
"
|
|
else
|
|
TESTMODE=""
|
|
fi
|
|
|
|
PWO_START(){
|
|
startup=true
|
|
while true; do
|
|
upsinfo=$(upsc CP1500PFCLCD)
|
|
if [ "$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')" != "" ]; then
|
|
OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //')
|
|
OFFSITE_LOAD=`echo "scale=2; ${OFFSITE_LOAD}/100" | bc`
|
|
OFFSITE_WATTS=`echo "scale=2; ${OFFSITE_LOAD}*$(echo "${upsinfo}" | grep 'ups.realpower.nominal:' | sed 's/.*: //')" | bc`
|
|
OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //')
|
|
OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc`
|
|
OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //')
|
|
SENDNOTICE(){
|
|
MESSAGE="${TESTMODE}${2}
|
|
|
|
BATTERY CHARGE: ${OFFSITE_BATT_CAP}%25
|
|
BATT RUNTIME: ${OFFSITE_BATT_RUNTIME}'mins
|
|
AC VOLTAGE: ${OFFSITE_VOLTIN}'volts"
|
|
if [ "${3}" = "1" ]; then PRIORITY="${3}"; MSGSOUND=siren; elif [ "${3}" != "" ]; then PRIORITY=${3}; else PRIORITY=0; fi
|
|
curl -sd "token=${PUSHOVER_APP_TOKEN}&user=${PUSHOVER_USER_TOKEN}&message=${MESSAGE}&title=${1}&priority=${PRIORITY}&sound=cosmic" https://api.pushover.net/1/messages.json >/dev/null 2>&1
|
|
}
|
|
if [ "${startup}" == "true" ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Service startup" >> ${PW_LOGPOWER}
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Service startup" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE POWER MONITOR" "Service starting"
|
|
startup=false
|
|
fi
|
|
|
|
if [ ${OFFSITE_VOLTIN%.*} -gt 105 ]; then
|
|
if [ -f ${PW_TMPFOLDER}/power.offsite.charging ] && [ ${OFFSITE_BATT_CAP%.*} -eq 100 ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Charging Complete" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE POWER" "Batteries are fully recharged"
|
|
rm -f ${PW_TMPFOLDER}/power.offsite.charging
|
|
elif [ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.charging ]; then
|
|
touch ${PW_TMPFOLDER}/power.offsite.charging
|
|
fi
|
|
if [ -f ${PW_TMPFOLDER}/power.offsite.ac.low ]; then
|
|
if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Offsite Power Restored" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE POWER" "Power restored! Will wait ${WAIT_TO_POWER_SERVERS}'mins before turning host(s) back on"
|
|
fi
|
|
|
|
rm -f ${PW_TMPFOLDER}/power.offsite.ac.*
|
|
fi
|
|
if [ -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/power.offsite.restored ]; then
|
|
touch ${PW_TMPFOLDER}/power.offsite.restored
|
|
|
|
elif [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.restored)) -gt $(echo "scale=2; ${WAIT_TO_POWER_SERVERS}*60" | bc) ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering Hosts ON After Power Outage" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE HOST POWER" "Powering offsite host(s) back ONLINE after power outage"
|
|
for host in ${PW_OFFSITEHOSTS}; do
|
|
if [ "$TEST" = "" ] || [ "$TEST" = "full" ]; then
|
|
ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on &
|
|
else
|
|
echo "TESTMODE: Powering up '${host}'"
|
|
fi
|
|
done
|
|
rm -f ${PW_TMPFOLDER}/power.offsite.*
|
|
fi
|
|
fi
|
|
logtime=$(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER}))
|
|
if [ ${logtime} -gt $(echo "scale=2; ${LOG_INTERVAL_NORMAL}*60" | bc) ] || ([ ${OFFSITE_BATT_CAP%.*} -lt 100 ] && [ ${logtime} -gt ${LOG_INTERVAL_CHARGING} ]); then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}% Battery - ${OFFSITE_BATT_RUNTIME}'mins - Normal" >> ${PW_LOGPOWER}
|
|
fi
|
|
else
|
|
[ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low ] && touch ${PW_TMPFOLDER}/power.offsite.ac.low
|
|
|
|
if [ ! -f ${PW_TMPFOLDER}/power.offsite.ac.low.notice ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low.notice)) -gt ${RENOTIFY_LOWPOWER} ]; then
|
|
# echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering LOW/OFF!" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE POWER - POWER OFF/LOW!!" "Power off or low voltage detected" 1
|
|
touch ${PW_TMPFOLDER}/power.offsite.ac.low.notice
|
|
fi
|
|
|
|
if ([ ${OFFSITE_BATT_CAP%.*} -lt ${SHUTDOWNSERVERS_AT_LOAD} ] || [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.offsite.ac.low)) -ge $(echo "scale=2; ${SHUTDOWNSERVERS_AFTER}*60" | bc) ]) && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Powering Down Hosts!" >> ${PW_LOGFILE}
|
|
SENDNOTICE "OFFSITE HOST POWER" "Powering DOWN offsite host(s) due to power outage" 1
|
|
touch ${PW_TMPFOLDER}/power.offsite.off
|
|
for host in ${PW_OFFSITEHOSTS}; do
|
|
if [ "$TEST" = "" ] || [ "$TEST" = "full" ]; then
|
|
ssh root@${host} /vmfs/volumes/652c94ef-ee49895b-365d-00109b3fea40/.SCRIPTS/esxi-shutdown2.sh &
|
|
else
|
|
echo "TESTMODE: Shutting down '${host}'"
|
|
fi
|
|
done
|
|
fi
|
|
if [ $(expr `date +%s` - $(stat -f %m ${PW_LOGPOWER})) -gt ${LOG_INTERVAL_ABNORMAL} ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_WATTS}'watts - ${OFFSITE_BATT_CAP}% Battery - ${OFFSITE_BATT_RUNTIME}'mins - OFF/LOW POWER!!" >> ${PW_LOGPOWER}
|
|
fi
|
|
fi
|
|
|
|
else
|
|
if [ $(expr `date +%s` - $(stat -f %m ${PW_TMPFOLDER}/power.ups.losscomm)) -ge 30 ]; then
|
|
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${OFFSITE_VOLTIN}'volts - ${OFFSITE_BATT_CAP}% Battery - Restarting NUT service" >> ${PW_LOGFILE}
|
|
SENDNOTICE "NUT DRIVER FAILURE" "Restarting NUT service"
|
|
/usr/local/etc/rc.d/nut.sh restart
|
|
sleep 10s
|
|
rm -f ${PW_TMPFOLDER}/power.ups.losscomm
|
|
|
|
elif [ ! -f ${PW_TMPFOLDER}/power.ups.losscomm ]; then
|
|
touch ${PW_TMPFOLDER}/power.ups.losscomm
|
|
fi
|
|
|
|
fi
|
|
sleep 5s
|
|
done
|
|
}
|
|
|
|
PWO_STOP(){
|
|
ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}' | xargs kill
|
|
}
|
|
|
|
|
|
PWO_RESTART() {
|
|
PWO_STOP
|
|
sleep 2s
|
|
/opt/idssys/powerwall/offsite-power-check.sh start &
|
|
|
|
}
|
|
|
|
# LOGROTATE(){
|
|
# lm=`expr $(date +'%m') - 1`
|
|
# if [ ${lm} = 0 ]; then
|
|
# lm=12
|
|
# ly=`expr $(date +'%Y') - 1`
|
|
# else
|
|
# ly=$(date +'%Y')
|
|
# fi
|
|
# [ ${lm} -lt 10 ] && lm="0${lm}"
|
|
# ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
|
|
# # mkdir -p ${ARCHIVEFOLDER}
|
|
# # mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null
|
|
# [ ! -f ${PW_LOGFILE} ] && touch ${PW_LOGFILE}
|
|
# [ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER}
|
|
# backupstokeep=($(ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort | tail -12))
|
|
# for folder in $(ls -d ${PW_LOGARCHIVE}/* | sort -r); do
|
|
# keep=0
|
|
# for filetokeep in ${backupstokeep[@]}; do
|
|
# echo "$folder $filetokeep"
|
|
# if [ "${folder}" == "${filetokeep}" ]; then
|
|
# keep=1; break
|
|
# fi
|
|
# done;
|
|
# [ ${keep} == 0 ] && rm -Rf ${folder}
|
|
# done
|
|
# }
|
|
|
|
case $1 in
|
|
start) PWO_START &;;
|
|
stop) PWO_STOP;;
|
|
restart) PWO_RESTART;;
|
|
# logrotate) LOGROTATE;;
|
|
update)
|
|
cd ${PW_FOLDER}
|
|
if [ "`git log --pretty=%H ...refs/heads/master^ | head -n 1`" != "`git ls-remote origin -h refs/heads/master |cut -f1`" ]; then
|
|
git fetch origin master >/dev/null 2>&1
|
|
git reset --hard origin/master >/dev/null 2>&1
|
|
git reflog expire --expire=now --all >/dev/null 2>&1
|
|
git repack -ad >/dev/null 2>&1
|
|
git prune >/dev/null 2>&1
|
|
git pull >/dev/null 2>&1
|
|
echo "Updated, restarting services"
|
|
/opt/idssys/powerwall/offsite-power-check.sh restart &
|
|
else
|
|
echo "No update available"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
|
|
exit 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|