voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-08-13 20:06:38 -05:00
parent c3ca15f017
commit 544c1ad389
2 changed files with 221 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
defaults.inc
View File

@@ -29,6 +29,11 @@ PW_LOGPOWER=${PW_LOGFOLDER}/log-power
[ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER}
declare -A PW_POWERWALL_SERVICES
PW_POWERWALL_SERVICES['temp']='monitor-temp'
PW_POWERWALL_SERVICES['power']='monitor-power'
# PW_POWERWALL_SERVICES['powerlogger']='monitor-powerlogger'
declare -A PW_TEMP_THRESHOLDS
# PW_TEMP_THRESHOLDS['ServerRoomTH']="72,74,76"
PW_TEMP_THRESHOLDS['ServerRoomTH']="83,88,95"
@@ -39,6 +44,11 @@ declare -A PW_SENSOR_TYPE
PW_SENSOR_TYPE['ServerRoomTH']='mqtt'
PW_SENSOR_TYPE['RaspberryPI-CPU']='system'
PW_SENSOR_TYPE['OctoPI-CPU']='remotesystem'
PW_SENSOR_TYPE['esxi0']='esxi'
PW_SENSOR_TYPE['esxi1']='esxi'
PW_SENSOR_TYPE['esxi2']='esxi'
PW_SENSOR_TYPE['esxi7']='esxi'
PW_SENSOR_TYPE['esxi8']='esxi'
declare -A PW_REMOTE_SENSORS
PW_REMOTE_SENSORS['ServerRoomTH']='10.10.20.10'
@@ -48,18 +58,25 @@ declare -A PW_SENSOR_ID
PW_SENSOR_ID['ServerRoomTH']='2'
PW_SENSOR_ID['RaspberryPI-CPU']='6'
PW_SENSOR_ID['OctoPI-CPU']='7'
# PW_SENSOR_ID['esxi0']='8'
# PW_SENSOR_ID['esxi1']='9'
# PW_SENSOR_ID['esxi2']='10'
# PW_SENSOR_ID['esxi7']='11'
# PW_SENSOR_ID['esxi8']='12'
declare -A PW_POWERWALL_SERVICES
PW_POWERWALL_SERVICES['temp']='monitor-temp'
PW_POWERWALL_SERVICES['power']='monitor-power'
# PW_POWERWALL_SERVICES['powerlogger']='monitor-powerlogger'
declare -A PW_ESXI_MON_HOSTS
PW_ESXI_MON_HOSTS['esxi0']='10.10.2.10'
PW_ESXI_MON_HOSTS['esxi1']='10.10.2.11'
PW_ESXI_MON_HOSTS['esxi2']='10.10.2.12'
PW_ESXI_MON_HOSTS['esxi7']='10.10.2.17'
PW_ESXI_MON_HOSTS['esxi8']='10.2.1.18'
declare -A PW_ESXI_HOST_NAMES
PW_ESXI_HOST_NAMES['10.10.2.10']='ids-vms-scsi.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.11']='ids-vms1.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.12']='ids-vms2.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.17']='ids-vms7.scity.us'
PW_ESXI_HOST_NAMES['10.2.1.18']='ids-vms-offsite.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.10']='iDS-vMS-scsi.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.11']='iDS-vMS1.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.12']='iDS-vMS2.scity.us'
PW_ESXI_HOST_NAMES['10.10.2.17']='iDS-vMS7.scity.us'
PW_ESXI_HOST_NAMES['10.2.1.18']='iDS-vMS-Offsite.scity.us'
PW_VCENTERHOST=10.10.1.30
PW_MAINHOSTS=10.10.2.10,10.10.2.11,10.10.2.12
@@ -68,6 +85,8 @@ PW_OFFSITEHOSTS=10.2.1.18
PW_MAINSITE_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS}"
PW_ALL_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS},${PW_OFFSITEHOSTS}"
PW_RACADM_ACCESS=10.10.0.241
IFS=,
PW_MAINSITE_HOSTS=(${PW_MAINSITE_HOSTS})
PW_MAINHOSTS=(${PW_MAINHOSTS})

319
powerwall.sh
View File

@@ -45,6 +45,7 @@ CHECKTEMP(){
fi
[ "${action}" != "" ] && echo
}
CHECKTEMPSENSOR(){
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
@@ -64,7 +65,36 @@ CHECKTEMPSENSOR(){
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
fi
elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then
for e in ${!PW_ESXI_MON_HOSTS[@]}; do
vHOSTIP=${PW_ESXI_MON_HOSTS[${e}]}
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTIP}]}
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS
for sensordata in "${vHOSTDATA[@]}"; do
sensor=$(echo ${sensordata} | cut -d'T' -f1 | xargs)
temp_c=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); current=${current/ /}
temp_warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); warn=${warn/ /}
temp_crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); crit=${crit/ /}
done
else #host down
if [ ! -f ${PW_TMPFOLDER}/${e}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${e}.down)) -ge 1800 ]; then
SENDNOTICE "${vHOSTIP} ERROR" "${vHOSTNAME} Host is down" 1
echo "($datetime) - ${vHOSTIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${e}.down
fi
fi
done
fi
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
@@ -120,145 +150,180 @@ CHECKTEMP_SERVICE(){
errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
[ $errtime -ge 3600 ] && relog=1 || relog=0
if [ "${PW_REMOTE_SENSORS[${SENSOR}]}" == "" ] || ([ "${PW_REMOTE_SENSORS[${SENSOR}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSOR}]})" != "false" ]); then
if [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then
SENDNOTICE "${SENSOR} Online" "${SENSOR} Sensor is back online"
echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is back onlione" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}.down
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ]
if [ "${PW_REMOTE_SENSORS[${SENSOR}]}" == "" ] || ([ "${PW_REMOTE_SENSORS[${SENSOR}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSOR}]})" != "false" ]); then
if [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then
SENDNOTICE "${SENSOR} Online" "${SENSOR} Sensor is back online"
echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is back onlione" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}.down
fi
fi
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
temp_c=$(vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
temp_h=''
fi
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
fi
datetime=`date +'%Y-%m-%d %H:%M:%S'`
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
last_temp[${SENSOR}]=0
temp_diff=0
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
[ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${PW_TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${PW_TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
fi
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
if [ "${last_temp[${SENSOR}]}" != "" ]; then
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
last_temp[${SENSOR}]=0
temp_diff=0
fi
${mysql_conn} -e "${QRY}"
touch ${PW_TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1
echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.warn
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
[ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${PW_TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" &
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*
elif [ $relog -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp}
fi
rm -f ${PW_TMPFOLDER}/temp.*
last_temp[${SENSOR}]=1
fi
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
wait=60
else
wait=300
fi
if [ ${lastinsert} -ge ${wait} ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${PW_TMPFOLDER}/${SENSOR}.insert
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1
echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
fi
fi
else #sensor down
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} ERROR" "${SENSOR} Sensor is down" 1
echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}.down
fi
fi
else
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then
SENDNOTICE "${SENSOR} ERROR" "${SENSOR} Sensor is down" 1
echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${SENSOR}.down
#############################
else #esxi hosts
for e in ${!PW_ESXI_MON_HOSTS[@]}; do
vHOSTIP=${PW_ESXI_MON_HOSTS[${e}]}
vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTIP}]}
if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS
for sensordata in "${vHOSTDATA[@]}"; do
sensor=$(echo ${sensordata} | cut -d'T' -f1 | xargs)
temp=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); current=${current/ /}
warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); warn=${warn/ /}
crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); crit=${crit/ /}
done
else #host down
if [ ! -f ${PW_TMPFOLDER}/${e}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${e}.down)) -ge 1800 ]; then
SENDNOTICE "${vHOSTIP} ERROR" "${vHOSTNAME} Host is down" 1
echo "($datetime) - ${vHOSTIP} - ${vHOSTNAME} Host is down" >> ${logtemp}
touch ${PW_TMPFOLDER}/${e}.down
fi
fi
done
fi
fi
done
@@ -268,6 +333,8 @@ CHECKTEMP_SERVICE(){
done # &
}
DAILYTEMP(){
# start=`date +%s`
cw=26;