Files
powerwall/powerwall.sh
2023-05-27 16:59:47 -05:00

797 lines
30 KiB
Bash
Executable File

#!/usr/bin/env bash
# powerwall - CLI commands to control VM guest power
action="${1}"
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/powerwall/defaults.inc
source ${FOLDER}/powerwall.conf
# TEST=true
shopt -s lastpipe
VERBOSE=false
CHECKTEMP(){
cw=20;
echo
# echo -e "${idsCL[Yellow]}Sensor Temperature(s) ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
echo -e "${idsCL[Yellow]}Sensor Temperature(s) Current/7DayAVG ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
DIVIDER . yellow 75
if [ "${1}" != "" ]; then
# reading=$(CHECKTEMPSENSOR ${1})
# c=0; spc=''; spc1=`expr ${cw} - ${#1}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
# echo -e "${idsCL[LightCyan]}${1}${spc}${idsCL[Default]}: ${reading}"
for SENSOR in ${!SENSOR_ID[@]}; do
if [[ "${SENSOR}" = *"${1}"* ]]; then
reading=$(CHECKTEMPSENSOR ${SENSOR})
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
fi
done
else
for SENSOR in ${!SENSOR_ID[@]}; do
reading=$(CHECKTEMPSENSOR ${SENSOR})
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
done
fi
[ "${action}" != "" ] && echo
}
CHECKTEMPSENSOR(){
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
average=$(AVERAGETEMP ${SENSOR_ID[${1}]} 7)
if [ "${SENSOR_TYPE[${1}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${SENSOR_TYPE[${1}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
fi
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
#echo "$temp_c -> $temp_f"
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
echo -en "${idsCL[Yellow]}WARNING (${temp_f}'F / ${average}'F)${idsCL[Default]}"
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
echo -en "${idsCL[Red]}CRITICAL (${temp_f}'F / ${average}'F)${idsCL[Default]}"
else
echo -en "${idsCL[Green]}Normal (${temp_f}'F / ${average}'F)${idsCL[Default]}"
fi
c=0; spc=''; spc1=`expr 10 - ${#temp_f} - 2`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
echo -e "${spc}[${spcw}${idsCL[Green]}<=$(expr ${temp_warn} - 1)'F${idsCL[Default]} /${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F${idsCL[Default]} ]"
fi
}
AVERAGETEMP(){
temptotal=0
daysback=${2}
sensorid=${1}
logsfound=0;
while IFS=$'\t' read date sensorid temp hum ;do
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=${sensorid} AND date BETWEEN
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
echo ${average}
}
CHECKTEMP_SERVICE(){
cw=20
declare -A last_temp
while true; do
for SENSOR in ${!SENSOR_ID[@]}; do
logtemp=${LOGFOLDER}/log-temp-${SENSOR}
[ ! -f ${logtemp} ] && touch ${logtemp}
errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
[ $errtime -ge 3600 ] && relog=1 || relog=0
if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${logfile}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
fi
[ -f ${TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${TMPFOLDER}/${SENSOR}.insert
fi
else
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
echo "($datetime) - alert sent" >> ${logtemp}
fi
done
sleep 10
done # &
}
CHECKPOWER(){
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
echo
echo -e "${idsCL[Yellow]}Power Information"
DIVIDER . yellow 75
echo -e "${idsCL[LightCyan]}Input Power ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}V / ${INPUTA}A"
echo -e "${idsCL[LightCyan]}Output Power ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
echo -e "${idsCL[LightCyan]}Battery Power ${idsCL[Default]}: ${idsCL[Green]}${BATTV}V / ${BATTA}A ${idsCL[Default]}"
[ "${action}" != "" ] && echo
}
CHECKPOWER_SERVICE(){
voltstatus=0
battstatus=0
# rm -f ${TMPFOLDER}/powerlogger.insert
while true; do
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
if [ "${INVSTATSINFO}" != "" ]; then
[ -f ${TMPFOLDER}/powerlogger.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/powerlogger.insert)) || lastinsert=100000
if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
POWERLOGGER "${INVSTATSINFO}" &
fi
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
if [ $voltstatus -eq 3 ]; then
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logpower}
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
VOLTAGE: ${INPUTACV}V"
echo "($datetime) alert sent" >> ${logpower}
echo -e "($datetime) - ${INPUTACV}V - POWER NOMINAL AGAIN" >> ${logfile}
battstatus=0
else
errtime=$(expr `date +%s` - $(stat -c %Y ${logpower}))
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logpower}
if [ $voltstatus -eq 0 ]; then
SENDNOTICE "Power Nominal" "($datetime) Service Startup
Normal voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V"
echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logpower}
echo -e "($datetime) - ${INPUTACV}V / ${BATTVOLT}V - Service Startup - Normal Voltage" >> ${logfile}
fi
fi
voltstatus=1
rm -f ${TMPFOLDER}/power.ac.*
else
echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logpower}
echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logfile}
if [ $voltstatus -lt 3 ]; then
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "($datetime) Power off or low voltage detected
VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V" 1
echo "($datetime) alert sent" >> ${logpower}
last_battvolt=$BATTVOLT
fi
voltstatus=3
touch ${TMPFOLDER}/power.ac.low
fi
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
Shutting down main servers" 1
SHUTDOWN_MAIN power &
battstatus=2
fi
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
Shutting down all servers" 1
SHUTDOWN_SYS power &
battstatus=3
fi
else
errtime=$(expr `date +%s` - $(stat -c %Y ${logpower}))
[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logpower}
if [ $battstatus -eq 0 ]; then
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logpower}
fi
rm -f ${TMPFOLDER}/power.dc.*
battstatus=1
fi
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
if [ $volt_diff -gt 0 ]; then
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logpower}
last_battvolt=$BATTVOLT
fi
else
sleep 20s
fi
sleep 10s
done # &
}
POWERLOGGER(){
INVSTATSINFO=${1}
datetime=`date +'%Y-%m-%d %H:%M:%S'`
# INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
touch ${TMPFOLDER}/powerlogger.insert
}
POWERLOGGER_SERVICE(){
while true; do
datetime=`date +'%Y-%m-%d %H:%M:%S'`
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
if [ "$(compgen -G "${TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
sleep 1m
else
sleep 5m
fi
done
}
RESTARTALL_SERVICES(){
STOPALL_SERVICES
sleep 2s
STARTALL_SERVICES
}
STOPALL_SERVICES(){
for srvc in "${!POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} stop
done
}
STARTALL_SERVICES(){
for srvc in "${!POWERWALL_SERVICES[@]}"; do
SERVICE_MNG ${srvc} start
done
}
SERVICE_MNG(){
if [ "${2}" = "start" ]; then
sa=Starting
elif [ "${2}" = "stop" ]; then
sa=Stopping
elif [ "${2}" = "restart" ]; then
sa=Restarting
elif [ "${2}" = "status" ]; then
sa='Status for'
fi
echo -en "${idsCL[LightCyan]}${sa} ${POWERWALL_SERVICES[${1}]} Service ... "
/bin/systemctl ${2} ${POWERWALL_SERVICES[${1}]}
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
}
STOP_SERVICE(){
echo "stop: ${1}"
service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}`
/usr/bin/pkill -P $service_pid
}
CHECK_SERVICES(){
echo
echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
DIVIDER . yellow 75
for srvc in "${POWERWALL_SERVICES[@]}"; do
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
done
echo
}
SHUTDOWN_MAIN(){
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS MAIN ${2}
reason='Server Room Overheated!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown
${reason}" 1
echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${logfile}
fi
echo
}
SHUTDOWN_SYS(){
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS SYS ${2}
reason='Server Room Overheated!'
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown
${reason}" 1
echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${logfile}
fi
# curl --data "cmd=PWRoff" http://10.10.0.61/cmd
echo
}
SHUTDOWN_CRIT(){
if [ "${1}" == "RaspberryPI-CPU" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) Powerwall System shutting down" 1
echo -e "($datetime) - CRITICAL TEMP - Powerwall System shutting down" >> ${LOGFOLDER}/log-temp-${1}
echo -e "($datetime) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${logfile}
/sbin/poweroff
fi
}
SHUTDOWN_SERVERS(){
# HOSTS=${1}HOSTS[@]
# for ESXIHOST in "${!HOSTS}"; do
# echo "Shutting Down: ${ESXIHOST}"
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
# done
echo "($datetime) - Shutting down ${1} servers" >> ${logfile}
[ $TEST = true ] && echo "RUNNING IN TEST MODE" >> ${logfile}
if [ "${1}" = "MAIN" ]; then
if [ ! -f $FOLDER/shutdown.main ]; then
touch $FOLDER/shutdown.main
echo -en "Disabling CLS in vCenter ... "
if [ $TEST != true ]; then
ssh root@${VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
ssh root@${VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
ssh root@${VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
fi
echo "Done"
for ESXIHOST in "${ESXIHOSTS[@]}"; do
echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
[ $TEST != true ] && ${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${ESXIHOSTS[@]}"; do
echo -en "Verifying iSCSI VM's stopped on: ${ESXIHOST} ... "
[ $TEST != true ] && ${FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
# sleep 1m
for ESXIHOST in "${MAINHOSTS[@]}"; do
echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
if [ $TEST != true ]; then
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo "Done"
done
sleep 1m
for ESXIHOST in "${MAINHOSTS[@]}"; do
echo -en "Shutting down host: ${ESXIHOST} ..."
if [ $TEST != true ]; then
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
fi
echo "Done"
done
for ESXIHOST in "${MAINHOSTS[@]}"; do
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
if [ $TEST != true ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
sleep 1
done
fi
echo "Done"
done
fi
elif [ "${1}" = "SYS" ]; then
[ ! -f $FOLDER/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
if [ ! -f $FOLDER/shutdown.sys ]; then
touch $FOLDER/shutdown.sys
for ESXIHOST in "${SYSHOSTS[@]}"; do
echo -en "Suspending DB, PiHole & vCenter VM's on: ${ESXIHOST} ... "
[ $TEST != true ] && ${FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1
echo "Done"
done
for ESXIHOST in "${SYSHOSTS[@]}"; do
echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
if [ $TEST != true ]; then
${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo "Done"
done
for ESXIHOST in "${SYSHOSTS[@]}"; do
echo -en "Shutting Down Host: ${ESXIHOST} ... "
if [ $TEST != true ]; then
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0'
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
fi
echo "Done"
done
for ESXIHOST in "${SYSHOSTS[@]}"; do
echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
if [ $TEST != true ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
sleep 1
done
fi
echo "Done"
done
fi
fi
}
SHUTDOWN_SERVER(){
echo "($datetime) - Shutting down ${1}" >> ${logfile}
echo -en "Enabling Maintenance mode and will wait for iSCSI VM's to begin migrating ... "
[ $TEST != true ] && ${FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXI_HOST_NAMES[${1}]} & >/dev/null 2>&1
sleep 45s
echo "Done"
echo -en "Shutting Down VM's on: ${1} ... "
if [ $TEST != true ]; then
${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${1} >/dev/null 2>&1
${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${1} >/dev/null 2>&1
fi
echo "Done"
echo -en "Shutting Down Host: ${1} ... "
if [ $TEST != true ]; then
ssh root@${1} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' & >/dev/null 2>&1
ssh root@${1} 'esxcli system maintenanceMode set -e false -t 0' >/dev/null 2>&1
fi
echo "Done"
echo -en "Waiting for Host (${1}) to shutdown ... "
if [ $TEST != true ]; then
while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
sleep 1
done
fi
echo "Done"
}
ISCSIVMSHUTDOWN(){
for ESXIHOST in "${ESXIHOSTS[@]}"; do
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... "
${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo -e "${idsCL[Green]}DONE${idsCL[Default]}"
echo
done
}
LOGROTATE(){
STOPALL_SERVICES
lm=`expr $(date +'%m') - 1`
if [ $lm = 0 ]; then
lm=12
ly=`expr $(date +'%Y') - 1`
else
ly=$(date +'%Y')
fi
ARCHIVEFOLDER=${LOGARCHIVE}/${ly}-${lm}
mkdir -p ${ARCHIVEFOLDER}
mv ${LOGFOLDER}/log* ${ARCHIVEFOLDER}/
source ${FOLDER}/defaults.inc
STARTALL_SERVICES
}
TEST(){
start=`date +%s`
temptotal=0
daysback=${1}
logsfound=0;
while IFS=$'\t' read date sensorid temp hum ;do
# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
# echo "- ${temptotal} + ${temp}"
((logsfound++))
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
echo "Logs found: ${logsfound}"
echo "Average: ${average}'F"
echo
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
echo
}
#########################################################
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
echo
DIVIDER . lightGreen 50
c=0; spc=''; spc1=`expr 18 - ${#VERS}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -e "${idsCL[LightGreen]}| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
DIVIDER . lightGreen 50
fi
# if [ ${action-x} ]; then
case $action in
services)
[ "${2}" == "stop" ] && STOPALL_SERVICES
[ "${2}" == "start" ] && STARTALL_SERVICES
[ "${2}" == "restart" ] && RESTARTALL_SERVICES
[ "${2}" == "status" ] && CHECK_SERVICES
;;
checktemp) CHECKTEMP ${2};;
checkpower) CHECKPOWER;;
test) CHECKTEMP ${2} ${3};;
logrotate) LOGROTATE;;
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
shutdownhost) SHUTDOWN_SERVER ${2};;
shutdown_servers) SHUTDOWN_SERVERS ${2};;
service)
if [ "${2}" = "all" ]; then
if [ "${3}" == "stop" ]; then
STOPALL_SERVICES
elif [ "${3}" == "start" ]; then
STARTALL_SERVICES
elif [ "${3}" == "restart" ]; then
RESTARTALL_SERVICES
elif [ "${3}" == "status" ]; then
CHECK_SERVICES
fi
else
if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
[ "${2}" = "temp" ] && CHECKTEMP_SERVICE
[ "${2}" = "power" ] && CHECKPOWER_SERVICE
[ "${2}" = "powerlogger" ] && POWERLOGGER_SERVICE
elif [ "${3}" = "stop" ]; then
if [ "${2}" = "temp" ]; then
STOP_SERVICE temp
for SENSOR in ${!SENSOR_ID[@]}; do
logtemp=${LOGFOLDER}/log-temp-${SENSOR}
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
done
elif [ "${2}" = "power" ]; then
STOP_SERVICE power
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logpower}
elif [ "${2}" = "powerlogger" ]; then
STOP_SERVICE powerlogger
fi
elif [ "${3}" = "restart" ]; then
SERVICE_MNG ${2} stop
slep 2s
SERVICE_MNG ${2} start
elif [ "${3}" = "status" ]; then
srvc=${POWERWALL_SERVICES[${2}]}
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
echo -e "${idsCL[Default]}"
echo
fi
fi
;;
update);;
status)
CHECKPOWER
CHECKTEMP
CHECK_SERVICES
;;
*)
echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}powerwall {option}"
echo
echo -e " ${idsCL[Yellow]}status ${idsCL[Default]}--> PowerWall Status Information"
echo -e " ${idsCL[Yellow]}checktemp {searchstring} ${idsCL[Default]}--> Check Temperature Sensors"
echo -e " ${idsCL[Yellow]}checkpower ${idsCL[Default]}--> Check Power Readings"
echo
# echo -e " ${idsCL[Yellow]}services {cmd} ${idsCL[Default]}--> Manage all monitor services at once"
# echo -e " ${idsCL[LightYellow]} cmd = {start/stop/restart/status}"
# echo
echo -e " ${idsCL[Yellow]}service {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service"
echo -e " ${idsCL[LightYellow]} srvc = { all / temp / power / powerlogger }"
echo -e " ${idsCL[LightYellow]} cmd = { start / stop / restart / status }"
echo
echo -e " ${idsCL[Yellow]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
echo
;;
esac
# fi
exit 0