update
This commit is contained in:
604
inc/temp.inc.sh
604
inc/temp.inc.sh
@@ -307,9 +307,9 @@ CHECKTEMP_SERVICE(){
|
||||
# touch ${PW_TMPFOLDER}/.sentdaily
|
||||
# SENDDAILY &
|
||||
# fi
|
||||
# fi
|
||||
# fi
|
||||
SENSORS_CHECKED=()
|
||||
for SENSORa in ${PW_SENSOR_ORD[@]}; do
|
||||
echo -en "${SENSORa}"; start_sub=`date +%s`
|
||||
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
|
||||
[ ! -f ${logtemp} ] && touch ${logtemp}
|
||||
[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0
|
||||
@@ -321,295 +321,9 @@ CHECKTEMP_SERVICE(){
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
|
||||
rm -f ${PW_TMPFOLDER}/${SENSORa}.down
|
||||
fi
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
|
||||
IFS=$'\n'
|
||||
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
|
||||
unset IFS
|
||||
[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
|
||||
t=1
|
||||
|
||||
else
|
||||
SENSORS_CHECK=(${SENSORa})
|
||||
fi
|
||||
if [ "${SENSORS_CHECK}" != "" ]; then
|
||||
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
||||
if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
|
||||
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
|
||||
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit
|
||||
[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
|
||||
fi
|
||||
CHECKTEMP_SERVICE_RUNSENSOR ${SENSORa} &
|
||||
SENSORS_CHECKED+=(${SENSORa})
|
||||
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
||||
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
||||
echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
||||
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
|
||||
INFO=$(curl -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq)
|
||||
if [[ "${SENSOR}" = *"-Probe" ]]; then
|
||||
temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes`
|
||||
else
|
||||
temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes`
|
||||
fi
|
||||
temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity')
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
||||
if [[ "${SENSOR}" = *"CPU"* ]]; then
|
||||
temp_c=$(vcgencmd measure_temp)
|
||||
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
||||
temp_h=''
|
||||
elif [[ "${SENSOR}" = *"FAN"* ]]; then
|
||||
temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input)
|
||||
temp_h=''
|
||||
fi
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
||||
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
||||
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
||||
temp_h=''
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
|
||||
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
|
||||
temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
|
||||
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
|
||||
temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
|
||||
temp_crit_sys=${temp_crit}
|
||||
temp_h=''
|
||||
vSENSOR=${SENSOR}
|
||||
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
||||
((t++))
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
|
||||
temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
|
||||
else
|
||||
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
|
||||
fi
|
||||
temp_h=''
|
||||
|
||||
fi
|
||||
[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
|
||||
|
||||
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
||||
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
GAUGE="CPU-USAGE"
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
GAUGE="RPM"
|
||||
else
|
||||
GAUGE="TEMP"
|
||||
fi
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
GAUGESH='%'
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
GAUGESH="'R"
|
||||
else
|
||||
GAUGESH="'F"
|
||||
fi
|
||||
[ -f ${PW_TMPFOLDER}/${SENSOR}.* ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.*
|
||||
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then
|
||||
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
|
||||
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.*
|
||||
|
||||
fi
|
||||
|
||||
if [ "${SENSOR}" == "ServerRoomTH" ]; then
|
||||
temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc`
|
||||
else
|
||||
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then
|
||||
temp_f=$(echo "scale=2; $temp_c/1" | bc -l)
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then
|
||||
temp_f=${temp_c}
|
||||
else
|
||||
temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32")
|
||||
fi
|
||||
(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
|
||||
fi
|
||||
|
||||
# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
|
||||
# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
|
||||
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
|
||||
fi
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
|
||||
vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
|
||||
# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
|
||||
if [ "${vHOSTFAN}" != "" ]; then
|
||||
FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`)
|
||||
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
|
||||
else
|
||||
FanSpeed=''
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
|
||||
if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
|
||||
[ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
|
||||
Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
|
||||
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
||||
|
||||
elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
|
||||
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
|
||||
# NORMAL TEMP: ${temp_f}${GAUGESH}"
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
|
||||
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
||||
|
||||
elif [ "${relog}" == "1" ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
|
||||
|
||||
fi
|
||||
rm -f ${PW_TMPFOLDER}/temp.*
|
||||
rm -f ${PW_TMPFOLDER}/power.${SENSORa}.*
|
||||
last_temp[${SENSOR//-/}]=1
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${FanSpeed}" != "" ]; then
|
||||
if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
|
||||
# if [ ${FanSpeed} -ne 30 ]; then
|
||||
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE}
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
|
||||
temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
|
||||
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
|
||||
|
||||
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.warn
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
|
||||
|
||||
fi
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
|
||||
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
|
||||
fi
|
||||
|
||||
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
|
||||
if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.crit
|
||||
SHUTDOWN_MAIN ${SENSOR} &
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
|
||||
|
||||
# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
|
||||
# SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
|
||||
# SHUTDOWN_SERVER ${SENSORa} -esxi off &
|
||||
# touch ${PW_TMPFOLDER}/power.${SENSORa}.off
|
||||
|
||||
fi
|
||||
fi
|
||||
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.critsys
|
||||
[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
|
||||
SHUTDOWN_CRIT ${SENSOR} &
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
|
||||
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
||||
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
|
||||
else
|
||||
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
|
||||
fi
|
||||
|
||||
# [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ] && temp_probe_f=$(bc <<< "scale=2; ${temp_probe_c}*1.8+32")
|
||||
|
||||
if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
||||
if [ "${entityid[1]}" == "" ]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
|
||||
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
||||
fi
|
||||
entityid=${entityid[1]}
|
||||
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
|
||||
fi
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
elif [ "${temp_h}" != "" ]; then
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
|
||||
else
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
fi
|
||||
${mysql_conn} -e "${QRY}"
|
||||
# if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
|
||||
# ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')"
|
||||
# fi
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}.insert
|
||||
fi
|
||||
|
||||
#### LOGGING OF SERVERROOM SENSOR DIFFERENCE
|
||||
if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then
|
||||
if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then
|
||||
if [ "${temp_f}" != "" ] && [ "${ServerRoomTH}" != "" ]; then
|
||||
[ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference
|
||||
temp_difference=$(bc <<< "scale=2; ${ServerRoomTH}-${temp_f}")
|
||||
# (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc`
|
||||
echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference
|
||||
fi
|
||||
fi
|
||||
elif [ "${SENSOR}" == "ServerRoomTH" ]; then
|
||||
export ServerRoomTH=${temp_f}
|
||||
fi
|
||||
###################
|
||||
|
||||
else
|
||||
[ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down
|
||||
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
||||
elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}-error.sent
|
||||
SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
# elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down
|
||||
else
|
||||
[ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down
|
||||
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then
|
||||
@@ -620,12 +334,320 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp}
|
||||
fi
|
||||
fi
|
||||
end_sub=`date +%s`; runtime_sub=$((end_sub-start_sub)); echo " - done: ${runtime_sub}s"
|
||||
done
|
||||
|
||||
for SENSORa in "${SENSORS_CHECKED[@]}"; do
|
||||
[ ! -f ${dli//.down/.disable} ] && SENSORS_CHECKED+=(${SENSORa})
|
||||
done
|
||||
|
||||
s_count=0; completed=false
|
||||
until [ "${completed}" == "true" ]; do
|
||||
for sensor in ${PW_TMPFOLDER}/zservicerun.*.done; do
|
||||
SENSORa=$(grep -oP '(?<=zservicerun.).*?(?=.done)' <<< "${nodestatus}")
|
||||
if [ "${SENSORa}" != "*" ]; then
|
||||
rm -f ${sensor}
|
||||
((s_count++))
|
||||
# echo "${NTS} == ${s_count} == ${nodestatus}"
|
||||
fi
|
||||
done
|
||||
[ ${s_count} -eq ${#SENSORS_CHECKED[@]} ] && completed=true
|
||||
done
|
||||
end=`date +%s`; runtime=$((end-start)); echo "Total runtime: ${runtime}s"
|
||||
# sleep 10s
|
||||
done # &
|
||||
${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
|
||||
}
|
||||
CHECKTEMP_SERVICE_RUNSENSOR(){
|
||||
SENSORa=${1}
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
|
||||
IFS=$'\n'
|
||||
read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"
|
||||
unset IFS
|
||||
[ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off
|
||||
t=1
|
||||
|
||||
else
|
||||
SENSORS_CHECK=(${SENSORa})
|
||||
fi
|
||||
if [ "${SENSORS_CHECK}" != "" ]; then
|
||||
for SENSOR in "${SENSORS_CHECK[@]}"; do
|
||||
if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then
|
||||
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn
|
||||
echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit
|
||||
[ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit}
|
||||
fi
|
||||
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then
|
||||
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
||||
echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
||||
echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
|
||||
INFO=$(curl -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq)
|
||||
if [[ "${SENSOR}" = *"-Probe" ]]; then
|
||||
temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes`
|
||||
else
|
||||
temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes`
|
||||
fi
|
||||
temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity')
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then
|
||||
if [[ "${SENSOR}" = *"CPU"* ]]; then
|
||||
temp_c=$(vcgencmd measure_temp)
|
||||
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
||||
temp_h=''
|
||||
elif [[ "${SENSOR}" = *"FAN"* ]]; then
|
||||
temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input)
|
||||
temp_h=''
|
||||
fi
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then
|
||||
temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp)
|
||||
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
||||
temp_h=''
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_c=`echo "scale=1; ${temp_c}/10" | bc`
|
||||
temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_warn=`echo "scale=1; ${temp_warn}/10" | bc`
|
||||
temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc)
|
||||
temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g')
|
||||
temp_crit=`echo "scale=1; ${temp_crit}/10" | bc`
|
||||
temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc)
|
||||
temp_crit_sys=${temp_crit}
|
||||
temp_h=''
|
||||
vSENSOR=${SENSOR}
|
||||
SENSOR="${SENSORa}-${vSENSOR// /_}"
|
||||
((t++))
|
||||
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then
|
||||
temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
|
||||
else
|
||||
temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage")
|
||||
fi
|
||||
temp_h=''
|
||||
|
||||
fi
|
||||
[ "${last_temp[${SENSOR//-/}]}" == "" ] && last_temp[${SENSOR//-/}]=0
|
||||
|
||||
logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR}
|
||||
if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
GAUGE="CPU-USAGE"
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
GAUGE="RPM"
|
||||
else
|
||||
GAUGE="TEMP"
|
||||
fi
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then
|
||||
GAUGESH='%'
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
GAUGESH="'R"
|
||||
else
|
||||
GAUGESH="'F"
|
||||
fi
|
||||
[ -f ${PW_TMPFOLDER}/${SENSOR}.* ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.*
|
||||
if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then
|
||||
SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again"
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp}
|
||||
rm -f ${PW_TMPFOLDER}/${SENSOR}-error.*
|
||||
|
||||
fi
|
||||
|
||||
if [ "${SENSOR}" == "ServerRoomTH" ]; then
|
||||
temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc`
|
||||
else
|
||||
if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then
|
||||
temp_f=$(echo "scale=2; $temp_c/1" | bc -l)
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then
|
||||
temp_f=${temp_c}
|
||||
else
|
||||
temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32")
|
||||
fi
|
||||
(( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}"
|
||||
fi
|
||||
|
||||
# echo "HERE: [${SENSOR//-/}] - ${last_temp[${SENSOR//-/}]} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}"
|
||||
# [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND"
|
||||
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
|
||||
echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
|
||||
fi
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
|
||||
vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq)
|
||||
# vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
|
||||
if [ "${vHOSTFAN}" != "" ]; then
|
||||
FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'`)
|
||||
# FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
|
||||
else
|
||||
FanSpeed=''
|
||||
fi
|
||||
fi
|
||||
|
||||
if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
|
||||
if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
|
||||
[ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
|
||||
Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
|
||||
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
||||
|
||||
elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
|
||||
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
|
||||
# NORMAL TEMP: ${temp_f}${GAUGESH}"
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
|
||||
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
|
||||
|
||||
elif [ "${relog}" == "1" ]; then
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp}
|
||||
|
||||
fi
|
||||
rm -f ${PW_TMPFOLDER}/temp.*
|
||||
rm -f ${PW_TMPFOLDER}/power.${SENSORa}.*
|
||||
last_temp[${SENSOR//-/}]=1
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${FanSpeed}" != "" ]; then
|
||||
if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then
|
||||
# if [ ${FanSpeed} -ne 30 ]; then
|
||||
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%25"
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE}
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
([ ! "${last_temp[${SENSOR//-/}]}" ] || (( $(bc <<< "${last_temp[${SENSOR//-/}]} < 2") ))) && last_temp[${SENSOR//-/}]=${temp_f}
|
||||
temp_diff=`echo "scale=2; ${last_temp[${SENSOR//-/}]}-${temp_f}" | bc`
|
||||
[ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc`
|
||||
|
||||
if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.warn
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}')" >> ${logtemp}
|
||||
|
||||
fi
|
||||
|
||||
# iDS-vMS-Offsite Fan Issue Workaround
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ ${FanSpeed} -ne 80 ]; then
|
||||
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%25" 1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1
|
||||
ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE}
|
||||
fi
|
||||
|
||||
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
|
||||
if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f touch ${PW_TMPFOLDER}/temp.crit ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.crit
|
||||
SHUTDOWN_MAIN ${SENSOR} &
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp}
|
||||
|
||||
# elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then
|
||||
# SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
|
||||
# SHUTDOWN_SERVER ${SENSORa} -esxi off &
|
||||
# touch ${PW_TMPFOLDER}/power.${SENSORa}.off
|
||||
|
||||
fi
|
||||
fi
|
||||
elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then
|
||||
if (( $(bc <<< "${temp_diff} > 1") )) || (( $(bc <<< "${temp_diff} = 1") )); then
|
||||
SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}')" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
|
||||
last_temp[${SENSOR//-/}]=${temp_f}
|
||||
if [ ! -f touch ${PW_TMPFOLDER}/temp.critsys ]; then
|
||||
touch ${PW_TMPFOLDER}/temp.critsys
|
||||
[ "${SENSOR}" == "ServerRoomTH" ] && SHUTDOWN_SYS ${SENSOR} TEMP &
|
||||
SHUTDOWN_CRIT ${SENSOR} &
|
||||
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp}
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
[ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000
|
||||
if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then
|
||||
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc)
|
||||
else
|
||||
wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc)
|
||||
fi
|
||||
|
||||
# [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ] && temp_probe_f=$(bc <<< "scale=2; ${temp_probe_c}*1.8+32")
|
||||
|
||||
if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then
|
||||
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
|
||||
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
||||
if [ "${entityid[1]}" == "" ]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')"
|
||||
entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'"))
|
||||
fi
|
||||
entityid=${entityid[1]}
|
||||
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')"
|
||||
fi
|
||||
elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then
|
||||
${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
elif [ "${temp_h}" != "" ]; then
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')"
|
||||
else
|
||||
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')"
|
||||
fi
|
||||
${mysql_conn} -e "${QRY}"
|
||||
# if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then
|
||||
# ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')"
|
||||
# fi
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}.insert
|
||||
fi
|
||||
|
||||
#### LOGGING OF SERVERROOM SENSOR DIFFERENCE
|
||||
if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then
|
||||
if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then
|
||||
if [ "${temp_f}" != "" ] && [ "${ServerRoomTH}" != "" ]; then
|
||||
[ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference
|
||||
temp_difference=$(bc <<< "scale=2; ${ServerRoomTH}-${temp_f}")
|
||||
# (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc`
|
||||
echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference
|
||||
fi
|
||||
fi
|
||||
elif [ "${SENSOR}" == "ServerRoomTH" ]; then
|
||||
export ServerRoomTH=${temp_f}
|
||||
fi
|
||||
###################
|
||||
|
||||
else
|
||||
[ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down
|
||||
if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}-error.reading
|
||||
elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
|
||||
touch ${PW_TMPFOLDER}/${SENSOR}-error.sent
|
||||
SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1
|
||||
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
touch ${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
|
||||
}
|
||||
|
||||
DAILYTEMP(){
|
||||
|
||||
@@ -79,7 +79,7 @@ CHECK_SERVICES(){
|
||||
}
|
||||
|
||||
SHUTDOWN_MAIN(){
|
||||
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
|
||||
if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
|
||||
touch ${PW_TMPFOLDER}/shutdown.MAIN.started
|
||||
SHUTDOWN_SERVER main -esxi off
|
||||
[ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!'
|
||||
@@ -98,7 +98,7 @@ ${reason}" 1
|
||||
echo
|
||||
}
|
||||
SHUTDOWN_SYS(){
|
||||
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
|
||||
if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
|
||||
if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ]; then
|
||||
touch ${PW_TMPFOLDER}/shutdown.MAINSITE.started
|
||||
SHUTDOWN_SERVER mainsite -esxi off
|
||||
|
||||
Reference in New Issue
Block a user