1064 lines
42 KiB
Bash
Executable File
1064 lines
42 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
action="${1}"
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/powerwall/defaults.inc
|
|
source ${PW_FOLDER}/settings.conf
|
|
|
|
# TEST=true
|
|
|
|
shopt -s lastpipe
|
|
|
|
VERBOSE=false
|
|
|
|
|
|
CHECKTEMP(){
|
|
cw=25;
|
|
echo
|
|
# echo -e "${idsCL[Yellow]}Sensor Temperature(s) ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
|
|
echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current / 3DayAVG ${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
|
|
DIVIDER . yellow 90
|
|
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${1}" == "" ] || [[ "${SENSOR}" = *"${1}"* ]]; then
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ]; then
|
|
vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]}
|
|
vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]}
|
|
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]}
|
|
echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[LightCyan]}- ${vHOSTiDRACIP}${idsCL[Default]}"
|
|
# echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..."
|
|
echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ..."
|
|
|
|
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
|
|
vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
|
|
IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS
|
|
echo -en "\r\033[K"
|
|
if [ ${#vHOSTDATA[@]} > 1 ]; then
|
|
for sensordata in "${vHOSTDATA[@]}"; do
|
|
vSENSOR=$(echo ${sensordata} | cut -d'T' -f1 | xargs)
|
|
temp_c=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); temp_c=${temp_c/ /}
|
|
temp_warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); temp_warn=${temp_warn/ /}
|
|
temp_crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); temp_crit=${temp_crit/ /}
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${vSENSOR// /_})
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#vSENSOR} - 1`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: ${reading}"
|
|
|
|
done
|
|
else
|
|
echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: No Data"
|
|
fi
|
|
|
|
else #host down
|
|
echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down"
|
|
|
|
# if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
|
|
# SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1
|
|
# echo "($datetime) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
|
|
# touch ${PW_TMPFOLDER}/${EHOST}.down
|
|
# fi
|
|
fi
|
|
|
|
else
|
|
echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..."
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR})
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}"
|
|
fi
|
|
echo
|
|
fi
|
|
done
|
|
|
|
[ "${action}" != "" ] && echo
|
|
}
|
|
|
|
CHECKTEMPSENSOR(){
|
|
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
# [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ] && average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} 3) || average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} 3 ${5})
|
|
|
|
if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then
|
|
if [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${1}]})" != "false" ]; then
|
|
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
fi
|
|
elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
|
|
temp_c=${2}
|
|
temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
|
|
fi
|
|
|
|
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
|
temp_f=$(bc <<< "scale=1; ${temp_c}*1.8+32")
|
|
#echo "$temp_c -> $temp_f"
|
|
c=0; spct=''; spc1=`expr 6 - ${#temp_f}`; until [ $c = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done
|
|
# c=0; spca=''; spc1=`expr 6 - ${#average}`; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done
|
|
#
|
|
# if [ "${average}" == "" ]; then
|
|
# averagedisp="${idsCL[Green]}"
|
|
# elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then
|
|
# averagedisp="${idsCL[Yellow]}${average}'F"
|
|
# elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then
|
|
# averagedisp="${idsCL[Red]}${average}'F"
|
|
# else
|
|
# averagedisp="${idsCL[Green]}${average}'F"
|
|
# fi
|
|
averagedisp=' '
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
echo -en "${idsCL[Yellow]}WARNING (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Yellow]})${idsCL[Default]}"
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
|
|
echo -en "${idsCL[Red]}CRITICAL (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Red]})${idsCL[Default]}"
|
|
else
|
|
echo -en "${idsCL[Green]}Normal (${spct}${temp_f}'F / ${spca}${averagedisp}${idsCL[Green]})${idsCL[Default]}"
|
|
fi
|
|
tmp="${spct}${temp_f}"
|
|
c=0; spc=''; spc1=`expr 8 - ${#tmp}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
|
|
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
|
|
echo -e "${spc}[${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F=>${idsCL[Default]} ]"
|
|
fi
|
|
}
|
|
AVERAGETEMP(){
|
|
temptotal=0
|
|
daysback=${2}
|
|
sensorid=${1}
|
|
logsfound=0;
|
|
if [ "${PW_SENSOR_TYPE[${sensorid}]}" == "esxi" ]; then
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'"))
|
|
entityid=${entityid[1]}
|
|
# echo "HERE: ${3} - ${entityid}"
|
|
WHERE="sensorid=${sensorid} AND entity=${entityid}"
|
|
else
|
|
WHERE="sensorid=${sensorid}"
|
|
fi
|
|
|
|
while IFS=$'\t' read date temp hum ;do
|
|
temptotal=$(bc <<< "scale=1; ${temptotal}+${temp}")
|
|
((logsfound++))
|
|
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
|
|
average=$(bc <<< "scale=1; ${temptotal}/${logsfound}")
|
|
echo ${average}
|
|
}
|
|
|
|
SENDDAILY(){
|
|
dailtemp=$(DAILYTEMP)
|
|
SENDNOTICE "Daily Temp Readings" "${dailtemp}"
|
|
}
|
|
|
|
CHECKTEMP_SERVICE(){
|
|
cw=20
|
|
declare -A last_temp
|
|
while true; do
|
|
|
|
if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then
|
|
if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then
|
|
touch ${PW_TMPFOLDER}/.sentdaily
|
|
SENDDAILY &
|
|
fi
|
|
fi
|
|
for SENSORa in ${PW_SENSOR_ORD[@]}; do
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
|
|
[ ! -f ${logtemp} ] && touch ${logtemp}
|
|
logtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
|
|
[ ${logtime} -ge 3600 ] && relog=1 || relog=0
|
|
|
|
# if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then
|
|
if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then
|
|
if [ -f ${PW_TMPFOLDER}/${SENSORa}.down ]; then
|
|
SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online"
|
|
echo "($datetime) - ${SENSORa} - Sensor is back online" >> ${logtemp}
|
|
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
|
|
|
|
fi
|
|
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi222" ]; then
|
|
unset IFS
|
|
SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
|
|
IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
|
|
# for i in ${SENSORS_CHECK[@]}; do echo $i; done
|
|
else
|
|
SENSORS_CHECK=(${SENSORa})
|
|
fi
|
|
|
|
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
temp_h=''
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
|
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
temp_h=''
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
temp_c=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); temp_c=${temp_c/ /}
|
|
temp_warn=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g');
|
|
temp_crit=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g');
|
|
temp_crit_sys=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g');
|
|
|
|
temp_warn=`echo $(bc <<< "scale=2; (${temp_warn/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
temp_crit=`echo $(bc <<< "scale=2; (${temp_crit/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
temp_crit_sys=`echo $(bc <<< "scale=2; (${temp_crit_sys/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'`
|
|
|
|
temp_h=''
|
|
vSENSOR=$(echo ${SENSOR} | cut -d'T' -f1 | xargs)
|
|
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
|
|
|
fi
|
|
|
|
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
|
|
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
|
|
echo "($datetime) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
|
|
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
|
|
|
fi
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
# echo "HERE: ${SENSOR} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
|
|
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
|
|
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
|
|
fi
|
|
|
|
if [ "${last_temp[${SENSOR}]}" != "" ]; then
|
|
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
|
|
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
|
|
else
|
|
last_temp[${SENSOR}]=0
|
|
temp_diff=0
|
|
fi
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
touch ${PW_TMPFOLDER}/temp.warn
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
|
|
last_temp[${SENSOR}]=$temp_f
|
|
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
|
|
fi
|
|
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
|
|
touch ${PW_TMPFOLDER}/temp.crit
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
|
|
Shutting down servers!!" 1
|
|
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
|
|
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
|
last_temp[${SENSOR}]=$temp_f
|
|
SHUTDOWN_MAIN ${SENSOR} &
|
|
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
|
|
|
|
fi
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
|
|
touch ${PW_TMPFOLDER}/temp.critsys
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
|
|
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
|
|
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
|
last_temp[${SENSOR}]=$temp_f
|
|
SHUTDOWN_SYS ${SENSOR} TEMP &
|
|
SHUTDOWN_CRIT ${SENSOR} &
|
|
echo -e "($datetime) - ${temp_f}'F - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
|
|
fi
|
|
else
|
|
|
|
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
|
|
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
|
|
Previous Temp: ${last_temp[${SENSOR}]}'F"
|
|
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
|
|
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
|
|
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
|
|
|
|
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
|
|
# SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
|
|
# NORMAL TEMP: ${temp_f}'F"
|
|
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
|
|
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
|
|
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
|
|
|
|
elif [ $relog -eq 1 ]; then
|
|
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
|
|
|
|
fi
|
|
rm -f ${PW_TMPFOLDER}/temp.*
|
|
last_temp[${SENSOR}]=1
|
|
fi
|
|
|
|
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
|
|
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
|
wait=60
|
|
else
|
|
wait=300
|
|
fi
|
|
if [ ${lastinsert} -ge ${wait} ]; then
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
|
if [ "${entity[1]}" == "" ]; then
|
|
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
|
|
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
|
fi
|
|
entityid=${entityid[1]}
|
|
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','${datetime}','${temp_f}')"
|
|
elif [ "${temp_h}" != "" ]; then
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}','${temp_h}')"
|
|
else
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${datetime}','${temp_f}')"
|
|
fi
|
|
${mysql_conn} -e "${QRY}"
|
|
touch ${PW_TMPFOLDER}/${SENSOR}.insert
|
|
fi
|
|
else
|
|
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
|
|
SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1
|
|
echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
|
|
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
|
|
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ]; then
|
|
SENDNOTICE "${SENSORa} ERROR" "${SENSORa} Sensor is down" 1
|
|
echo "($datetime) - ${SENSORa} - ${SENSORa} Sensor is down" >> ${logtemp}
|
|
touch ${PW_TMPFOLDER}/${SENSORa}.down
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
sleep 10
|
|
|
|
done # &
|
|
}
|
|
|
|
|
|
|
|
DAILYTEMP(){
|
|
cw=35;
|
|
echo
|
|
echo -e "Sensor Temperature(s) Now / 1DayAVG / 7DayAVG"
|
|
DIVIDER . . 60
|
|
for SENSORa in ${PW_SENSOR_ORD[@]}; do
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
unset IFS
|
|
SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
|
|
IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
|
|
else
|
|
SENSORS_CHECK=(${SENSORa})
|
|
fi
|
|
|
|
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
|
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
|
temp_c=$(echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1)
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
|
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
temp_c=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g');
|
|
temp_c=${temp_c/ /}
|
|
vSENSOR=$(echo ${SENSOR} | cut -d'T' -f1 | xargs)
|
|
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
|
|
|
else
|
|
temp_c=''
|
|
|
|
fi
|
|
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
|
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_})
|
|
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_})
|
|
else
|
|
average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1)
|
|
average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7)
|
|
fi
|
|
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
echo -e "${SENSOR}${spc}${temp_f}'F / ${average1}'F / ${average7}'F"
|
|
|
|
else
|
|
echo -e "${SENSOR}${spc}No Data"
|
|
|
|
fi
|
|
done
|
|
[ "$temp_c" != "null" ] && [ "$temp_c" != "" ] && echo
|
|
done
|
|
}
|
|
|
|
|
|
|
|
CHECKPOWER(){
|
|
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
|
|
INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
|
|
INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
|
|
OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
|
|
OUTPUTA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
|
|
BATTV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
|
|
BATTA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
|
|
|
|
OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
|
|
OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
|
|
|
|
echo
|
|
echo -e "${idsCL[Yellow]}Power Information"
|
|
DIVIDER . yellow 75
|
|
|
|
echo -e "${idsCL[LightCyan]}Input Power ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}V / ${INPUTA}A"
|
|
echo -e "${idsCL[LightCyan]}Output Power ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
|
|
echo -e "${idsCL[LightCyan]}Battery Power ${idsCL[Default]}: ${idsCL[Green]}${BATTV}V / ${BATTA}A ${idsCL[Default]}"
|
|
[ "${action}" != "" ] && echo
|
|
}
|
|
|
|
CHECKPOWER_SERVICE(){
|
|
voltstatus=0
|
|
battstatus=0
|
|
# rm -f ${PW_TMPFOLDER}/powerlogger.insert
|
|
while true; do
|
|
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
|
|
if [ "${INVSTATSINFO}" != "" ]; then
|
|
[ -f ${PW_TMPFOLDER}/powerlogger.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/powerlogger.insert)) || lastinsert=100000
|
|
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
|
wait=60
|
|
else
|
|
wait=300
|
|
fi
|
|
if [ ${lastinsert} -ge ${wait} ]; then
|
|
POWERLOGGER "${INVSTATSINFO}" &
|
|
fi
|
|
|
|
INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV')
|
|
BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV')
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
|
|
if [ $voltstatus -eq 3 ]; then
|
|
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${PW_LOGPOWER}
|
|
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}V"
|
|
echo -e "($datetime) - ${INPUTACV}V - POWER NOMINAL AGAIN" >> ${PW_LOGFILE}
|
|
battstatus=0
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${PW_LOGPOWER}
|
|
if [ $voltstatus -eq 0 ]; then
|
|
SENDNOTICE "Power Nominal" "($datetime) Service Startup
|
|
Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}V
|
|
BATT VOLTAGE: ${BATTVOLT}V"
|
|
echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
|
|
echo -e "($datetime) - ${INPUTACV}V / ${BATTVOLT}V - Service Startup - Normal Voltage" >> ${PW_LOGFILE}
|
|
fi
|
|
fi
|
|
voltstatus=1
|
|
rm -f ${PW_TMPFOLDER}/power.ac.*
|
|
else
|
|
echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${PW_LOGPOWER}
|
|
echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${PW_LOGFILE}
|
|
if [ $voltstatus -lt 3 ]; then
|
|
SENDNOTICE "POWER ALERT - POWER OFF/LOW POWER!!" "($datetime) Power off or low voltage detected
|
|
VOLTAGE: ${INPUTACV}V
|
|
BATT VOLTAGE: ${BATTVOLT}V" 1
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
voltstatus=3
|
|
touch ${PW_TMPFOLDER}/power.ac.low
|
|
fi
|
|
|
|
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
|
|
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
|
|
touch ${PW_TMPFOLDER}/power.dc.low
|
|
if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${PW_LOGPOWER}
|
|
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
|
|
Shutting down main servers" 1
|
|
SHUTDOWN_MAIN power &
|
|
battstatus=2
|
|
fi
|
|
|
|
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${PW_LOGPOWER}
|
|
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${PW_LOGFILE}
|
|
touch ${PW_TMPFOLDER}/power.dc.low
|
|
if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${PW_LOGPOWER}
|
|
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
|
|
Shutting down all servers" 1
|
|
SHUTDOWN_SYS power &
|
|
|
|
battstatus=3
|
|
|
|
fi
|
|
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${PW_LOGPOWER}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${PW_LOGPOWER}
|
|
if [ $battstatus -eq 0 ]; then
|
|
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${PW_LOGPOWER}
|
|
fi
|
|
rm -f ${PW_TMPFOLDER}/power.dc.*
|
|
battstatus=1
|
|
|
|
fi
|
|
|
|
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
|
|
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
|
|
if [ $volt_diff -gt 0 ]; then
|
|
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
|
|
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${PW_LOGPOWER}
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
else
|
|
sleep 20s
|
|
fi
|
|
|
|
if [ "${archivedone}" != "complete" ] && [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0000 ] && [ $(date +%H)$(date +%M) -le 0005 ]; then
|
|
LOGROTATE &
|
|
archivedone=complete
|
|
|
|
# elif [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0100 ] && [ "${archivedone}" = "complete" ]; then
|
|
# archivedone=false
|
|
|
|
fi
|
|
|
|
sleep 10s
|
|
done # &
|
|
}
|
|
|
|
POWERLOGGER(){
|
|
INVSTATSINFO=${1}
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
# INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
|
|
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
|
|
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
|
|
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
|
|
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
|
|
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
|
|
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
touch ${PW_TMPFOLDER}/powerlogger.insert
|
|
}
|
|
|
|
POWERLOGGER_SERVICE(){
|
|
while true; do
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
|
|
inV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
|
|
inA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
|
|
outV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
|
|
outA=`echo ${INVSTATSINFO} | jq '.outputs .outA'`
|
|
battV=`echo ${INVSTATSINFO} | jq '.inputs .battV'`
|
|
xfA=`echo ${INVSTATSINFO} | jq '.inputs .xfA'`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
|
|
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
|
sleep 1m
|
|
else
|
|
sleep 5m
|
|
fi
|
|
|
|
done
|
|
}
|
|
|
|
|
|
|
|
|
|
RESTARTALL_SERVICES(){
|
|
STOPALL_SERVICES
|
|
sleep 2s
|
|
STARTALL_SERVICES
|
|
}
|
|
STOPALL_SERVICES(){
|
|
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
|
|
SERVICE_MNG ${srvc} stop
|
|
done
|
|
}
|
|
STARTALL_SERVICES(){
|
|
for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do
|
|
SERVICE_MNG ${srvc} start
|
|
done
|
|
}
|
|
SERVICE_MNG(){
|
|
if [ "${2}" = "start" ]; then
|
|
sa=Starting
|
|
|
|
elif [ "${2}" = "stop" ]; then
|
|
sa=Stopping
|
|
|
|
elif [ "${2}" = "restart" ]; then
|
|
sa=Restarting
|
|
|
|
elif [ "${2}" = "status" ]; then
|
|
sa='Status for'
|
|
|
|
fi
|
|
echo -en "${idsCL[LightCyan]}${sa} ${PW_POWERWALL_SERVICES[${1}]} Service ... "
|
|
/bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]}
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
}
|
|
STOP_SERVICE(){
|
|
echo "stop: ${1}"
|
|
service_pid=`systemctl show --property MainPID --value ${PW_POWERWALL_SERVICES[${1}]}`
|
|
/usr/bin/pkill -P $service_pid
|
|
}
|
|
|
|
CHECK_SERVICES(){
|
|
echo
|
|
echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status"
|
|
DIVIDER . yellow 75
|
|
|
|
for srvc in "${PW_POWERWALL_SERVICES[@]}"; do
|
|
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
|
|
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
|
|
echo -e "${idsCL[Default]}"
|
|
done
|
|
|
|
echo
|
|
}
|
|
|
|
SHUTDOWN_MAIN(){
|
|
if [ "${1}" == "ServerRoomTH"]; then
|
|
SHUTDOWN_SERVERS MAIN ${2}
|
|
reason='Server Room Overheated!'
|
|
sendnotice=true
|
|
|
|
else
|
|
sendnotice=false
|
|
fi
|
|
|
|
if [ "${sendnotice}" == "true"]; then
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown
|
|
${reason}" 1
|
|
echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
|
|
fi
|
|
|
|
echo
|
|
}
|
|
SHUTDOWN_SYS(){
|
|
if [ "${1}" == "ServerRoomTH"]; then
|
|
SHUTDOWN_SERVERS SYS ${2}
|
|
reason='Server Room Overheated!'
|
|
fi
|
|
|
|
if [ "${sendnotice}" == "true"]; then
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown
|
|
${reason}" 1
|
|
echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE}
|
|
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
|
|
fi
|
|
|
|
# curl --data "cmd=PWRoff" http://10.10.0.61/cmd
|
|
echo
|
|
}
|
|
SHUTDOWN_CRIT(){
|
|
if [ "${1}" == "Powerwall-CPU" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) Powerwall System shutting down" 1
|
|
echo -e "($datetime) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
|
|
echo -e "($datetime) - ${SENSOR} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE}
|
|
# /sbin/poweroff
|
|
|
|
elif [ "${1}" == "OctoPI-CPU" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) OctoPI System shutting down" 1
|
|
echo -e "($datetime) - CRITICAL TEMP - OctoPI System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1}
|
|
echo -e "($datetime) - ${SENSOR} TEMP CRITICAL - OctoPI System shutting down" >> ${PW_LOGFILE}
|
|
# ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff
|
|
|
|
fi
|
|
}
|
|
|
|
SHUTDOWN_SERVERS(){
|
|
|
|
# HOSTS=${1}HOSTS[@]
|
|
# for ESXIHOST in "${!HOSTS}"; do
|
|
# echo "Shutting Down: ${ESXIHOST}"
|
|
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# done
|
|
|
|
echo "($datetime) - Shutting down ${1} servers" >> ${PW_LOGFILE}
|
|
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
|
|
|
|
if [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "ALL" ]; then
|
|
if [ ! -f $FOLDER/shutdown.main ]; then
|
|
touch $FOLDER/shutdown.main
|
|
echo -en "Disabling CLS in vCenter ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
|
|
ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
|
|
ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
|
|
fi
|
|
echo "Done"
|
|
|
|
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
|
|
echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
|
|
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
|
|
echo -en "Verifying iSCSI VM's stopped on: ${ESXIHOST} ... "
|
|
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
# sleep 1m
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
|
|
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
fi
|
|
echo "Done"
|
|
done
|
|
sleep 1m
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
echo -en "Shutting down host: ${ESXIHOST} ..."
|
|
if [ "${TEST}" != "true" ]; then
|
|
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
|
|
fi
|
|
echo "Done"
|
|
done
|
|
|
|
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
fi
|
|
echo "Done"
|
|
done
|
|
fi
|
|
fi
|
|
if [ "${1^^}" == "SYS" ] || [ "${1^^}" == "ALL" ]; then
|
|
[ ! -f $FOLDER/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
|
|
if [ ! -f $FOLDER/shutdown.sys ]; then
|
|
touch $FOLDER/shutdown.sys
|
|
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
echo -en "Suspending DB, PiHole & vCenter VM's on: ${ESXIHOST} ... "
|
|
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
fi
|
|
echo "Done"
|
|
done
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
echo -en "Shutting Down Host: ${ESXIHOST} ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0'
|
|
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
|
|
fi
|
|
echo "Done"
|
|
done
|
|
|
|
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
fi
|
|
echo "Done"
|
|
done
|
|
fi
|
|
fi
|
|
}
|
|
SHUTDOWN_SERVER(){
|
|
echo "($datetime) - Shutting down ${1}" >> ${PW_LOGFILE}
|
|
|
|
echo -en "Enabling Maintenance mode and will wait for iSCSI VM's to begin migrating ... "
|
|
[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_ESXI_HOST_NAMES[${1}]} & >/dev/null 2>&1
|
|
sleep 45s
|
|
echo "Done"
|
|
|
|
echo -en "Shutting Down VM's on: ${1} ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${1} >/dev/null 2>&1
|
|
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${1} >/dev/null 2>&1
|
|
fi
|
|
echo "Done"
|
|
|
|
echo -en "Shutting Down Host: ${1} ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
ssh root@${1} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' & >/dev/null 2>&1
|
|
ssh root@${1} 'esxcli system maintenanceMode set -e false -t 0' >/dev/null 2>&1
|
|
fi
|
|
echo "Done"
|
|
|
|
echo -en "Waiting for Host (${1}) to shutdown ... "
|
|
if [ "${TEST}" != "true" ]; then
|
|
while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
fi
|
|
echo "Done"
|
|
}
|
|
|
|
ISCSIVMSHUTDOWN(){
|
|
for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
|
|
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}' that are running on an iSCSI drive ... "
|
|
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
|
|
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
|
|
echo
|
|
done
|
|
|
|
echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..."
|
|
echo
|
|
}
|
|
|
|
LOGROTATE(){
|
|
# STOPALL_SERVICES
|
|
lm=`expr $(date +'%m') - 1`
|
|
if [ $lm = 0 ]; then
|
|
lm=12
|
|
ly=`expr $(date +'%Y') - 1`
|
|
else
|
|
ly=$(date +'%Y')
|
|
fi
|
|
ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm}
|
|
mkdir -p ${ARCHIVEFOLDER}
|
|
mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/
|
|
# source ${PW_FOLDER}/defaults.inc
|
|
# STARTALL_SERVICES
|
|
}
|
|
|
|
GET_SNAPSHOTS(){
|
|
# for ESXIHOST in "${!PW_ESXI_HOST_NAMES[@]}"; do
|
|
for SENSOR in ${PW_SENSOR_ORD[@]}; do
|
|
if [ "${PW_HOST_ADDRESSES[${SENSOR}]}" != "" ]; then
|
|
ESXIHOST=${PW_HOST_ADDRESSES[${SENSOR}]}
|
|
echo -e "${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]} (${ESXIHOST}) "
|
|
DIVIDER . lightCyan 70
|
|
echo -en "${idsCL[LighGreen]}Looking for snapshots...${idsCL[Default]}"
|
|
result=`${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 ${ESXIHOST}`
|
|
echo -e "\r\033[K"
|
|
[ "${result}" != "" ] && echo -e "${result}" || echo "No snapshots found"
|
|
DIVIDER true lightCyan 70
|
|
fi
|
|
done
|
|
}
|
|
|
|
TEST(){
|
|
start=`date +%s`
|
|
temptotal=0
|
|
daysback=${1}
|
|
logsfound=0;
|
|
while IFS=$'\t' read date sensorid temp hum ;do
|
|
# echo date:$date sensorid:$sensorid temp:$temp hum:$hum
|
|
temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}")
|
|
# echo "- ${temptotal} + ${temp}"
|
|
((logsfound++))
|
|
|
|
done < <(${mysql_conn} -se "USE servermonitor; SELECT date,sensorid,temp,hum from sensor_data WHERE sensorid=2 AND date BETWEEN
|
|
DATE_SUB(DATE(NOW()), INTERVAL ${daysback} DAY) AND NOW() ORDER BY id DESC;")
|
|
|
|
average=$(bc <<< "scale=2; ${temptotal}/${logsfound}")
|
|
|
|
echo "Logs found: ${logsfound}"
|
|
echo "Average: ${average}'F"
|
|
echo
|
|
|
|
# QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${INVSTATSINFO}')"
|
|
# ${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "runtime: ${runtime}"
|
|
echo
|
|
}
|
|
|
|
|
|
|
|
#########################################################
|
|
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
|
|
echo
|
|
DIVIDER . lightGreen 70
|
|
msg="| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})"
|
|
c=0; spc=''; spc1=`expr 81 - ${#msg}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
|
|
DIVIDER . lightGreen 70
|
|
fi
|
|
|
|
# if [ ${action-x} ]; then
|
|
case $action in
|
|
services)
|
|
[ "${2}" == "stop" ] && STOPALL_SERVICES
|
|
[ "${2}" == "start" ] && STARTALL_SERVICES
|
|
[ "${2}" == "restart" ] && RESTARTALL_SERVICES
|
|
[ "${2}" == "status" ] && CHECK_SERVICES
|
|
;;
|
|
checktemp) CHECKTEMP ${2};;
|
|
dailytemp)
|
|
start=`date +%s`
|
|
DAILYTEMP
|
|
# SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)"
|
|
end=`date +%s`; runtime=$((end-start))
|
|
echo -e "\nruntime: ${runtime}\n"
|
|
;;
|
|
checkpower) CHECKPOWER;;
|
|
test) CHECKTEMP ${2} ${3};;
|
|
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
|
|
|
|
shutdownhost) SHUTDOWN_SERVER ${2};;
|
|
shutdown_servers) SHUTDOWN_SERVERS ${2};;
|
|
|
|
get-snapshots) GET_SNAPSHOTS;;
|
|
|
|
service)
|
|
if [ "${2}" = "all" ]; then
|
|
if [ "${3}" == "stop" ]; then
|
|
STOPALL_SERVICES
|
|
elif [ "${3}" == "start" ]; then
|
|
STARTALL_SERVICES
|
|
elif [ "${3}" == "restart" ]; then
|
|
RESTARTALL_SERVICES
|
|
elif [ "${3}" == "status" ]; then
|
|
CHECK_SERVICES
|
|
fi
|
|
else
|
|
if [ "${3}" = "start" ] || [ "${3}" = "" ]; then
|
|
if [ "${2}" = "temp" ]; then
|
|
SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" &
|
|
CHECKTEMP_SERVICE
|
|
elif [ "${2}" = "power" ]; then
|
|
CHECKPOWER_SERVICE
|
|
elif [ "${2}" = "powerlogger" ]; then
|
|
POWERLOGGER_SERVICE
|
|
fi
|
|
|
|
elif [ "${3}" = "stop" ]; then
|
|
if [ "${2}" = "temp" ]; then
|
|
STOP_SERVICE temp
|
|
for SENSOR in ${!PW_SENSOR_ID[@]}; do
|
|
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
|
|
done
|
|
|
|
elif [ "${2}" = "power" ]; then
|
|
STOP_SERVICE power
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER}
|
|
|
|
elif [ "${2}" = "powerlogger" ]; then
|
|
STOP_SERVICE powerlogger
|
|
|
|
fi
|
|
|
|
elif [ "${3}" = "restart" ]; then
|
|
SERVICE_MNG ${2} stop
|
|
slep 2s
|
|
SERVICE_MNG ${2} start
|
|
|
|
elif [ "${3}" = "status" ]; then
|
|
srvc=${PW_POWERWALL_SERVICES[${2}]}
|
|
c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en "${idsCL[LightCyan]}${srvc}${spc}: "
|
|
[ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running"
|
|
echo -e "${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
fi
|
|
;;
|
|
|
|
update);;
|
|
status)
|
|
CHECKPOWER
|
|
CHECKTEMP
|
|
CHECK_SERVICES
|
|
;;
|
|
*)
|
|
echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}powerwall {option}"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}status ${idsCL[Default]}--> PowerWall Status Information"
|
|
echo -e " ${idsCL[Yellow]}checktemp {searchstring} ${idsCL[Default]}--> Check Temperature Sensors"
|
|
echo -e " ${idsCL[Yellow]}checkpower ${idsCL[Default]}--> Check Power Readings"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}shutdownhost {host-ip} ${idsCL[Default]}--> Shutdown host server"
|
|
echo -e " ${idsCL[Yellow]}iscsi-vm-shutdown ${idsCL[Default]}--> Shutdown all VM's running on an iSCSI Datastore"
|
|
echo -e " ${idsCL[Yellow]}shutdown_servers {type} ${idsCL[Default]}--> Shutdown selection of servers"
|
|
echo -e " ${idsCL[LightYellow]} type = { main / sys / all }"
|
|
echo
|
|
# echo
|
|
# echo -e " ${idsCL[Yellow]}services {cmd} ${idsCL[Default]}--> Manage all monitor services at once"
|
|
# echo -e " ${idsCL[LightYellow]} cmd = {start/stop/restart/status}"
|
|
# echo
|
|
echo -e " ${idsCL[Yellow]}service {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service"
|
|
echo -e " ${idsCL[LightYellow]} srvc = { all / temp / power / powerlogger }"
|
|
echo -e " ${idsCL[LightYellow]} cmd = { start / stop / restart / status }"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates"
|
|
echo
|
|
;;
|
|
esac
|
|
# fi
|
|
|
|
exit 0 |