GET_OFFSITEHOST_FANSPEED(){ vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') echo ${FanSpeed} > ${PW_TMPFOLDER}/.fanspeed } GET_HOST_FANSPEED(){ vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${1}]}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $9}' | sed -e 's/%//g') else FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') fi echo ${FanSpeed} > ${PW_TMPFOLDER}/.fanspeed.${PW_ESXI_HOST_NAMES[${1}]} } START_POWERWALL_FAN(){ pwmconfig & sleep 4s killall pwmconfig temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) if [ ${temp_c} -gt 0 ]; then SENDNOTICE "Starting up the ${SENSOR}" "Startup confirmed, the fan is now reading: ${temp_c}" else SENDNOTICE "FAILED to start ${SENSOR}" "Cannot start the fan, ERROR WILL ROBINSON!!!" fi } CHECKTEMP(){ start=`date +%s` cw=25; while [ $# -gt 0 ]; do case "$1" in -a|-avg) avgdays=${2};; -S|-sensors) temptype="sensors";; -e|-esxi) temptype="esxi";; -s|-search) search=${2};; esac shift 1 done if [ "${temptype}" != "sensors" ]; then for ESXIHOST in "${PW_ALLHOSTS[@]}"; do if [ ! -f ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.down ]; then GET_HOST_FANSPEED ${ESXIHOST} & fi done fi echo if [ "${avgdays}" != "" ]; then if [ $(bc -l <<< "${avgdays} < 1") -eq 1 ]; then avgdays_disp="$(ROUND_NUMBER $(bc <<< "scale=2; ${avgdays}*24*60") 0)Min" elif [ $(bc -l <<< "${avgdays} < 10") -eq 1 ]; then avgdays_disp=" ${avgdays}Day" else avgdays_disp="${avgdays}Day" fi echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current / ${idsCL[LightYellow]}${avgdays_disp}AVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" else avgdays=noavg echo -e "${idsCL[Cyan]}Sensor Temperature(s) Current ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" fi DIVIDER . yellow 70 for SENSOR in ${PW_SENSOR_ORD[@]}; do if [ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]; then if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ] && [ "${temptype}" != "sensors" ]; then vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]} vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]} vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]} echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]" if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then echo -en "${idsCL[LightCyan]} Getting FanSpeed..." RPM=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSOR}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) [ ! -f ${PW_TMPFOLDER}/.fanspeed.${SENSOR} ] && until [ -f ${PW_TMPFOLDER}/.fanspeed.${SENSOR} ]; do tmp=tmp; done echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]" echo -e "${idsCL[LightCyan]} [FanSpeed=$(cat ${PW_TMPFOLDER}/.fanspeed.${SENSOR})'% ($(IDS_NUMBER_FORMAT ${RPM})'RPM)]${idsCL[Default]}" else echo -e "${idsCL[Default]}" fi DIVIDER . lightCyan 70 echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... " # if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') IFS=$'\n' read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}" unset IFS echo -en "\r\033[K" if [ ${#vHOST_TEMPNAMES[@]} -gt 1 ]; then t=1 for sensorname in "${vHOST_TEMPNAMES[@]}"; do temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g') temp_c=`echo "scale=1; ${temp_c}/10" | bc` temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g') temp_warn=`echo "scale=1; ${temp_warn}/10" | bc` temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g') temp_crit=`echo "scale=1; ${temp_crit}/10" | bc` c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done [ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Calculating ${avgdays} Day average ... " reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays}) [ "${avgdays}" != "noavg" ] && echo -en "\r\033[K" echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}" ((t++)) done else echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data" fi else #host down echo -e "\r\033[K${idsCL[LightRed]}${SENSOR} Host is down${idsCL[Default]}" # if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then # SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1 # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp} # touch ${PW_TMPFOLDER}/${EHOST}.down # fi fi echo fi if [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ] && [ "${temptype}" != "esxi" ]; then [ "${last_sensor}" == "${SENSOR%%-*}" ] && echo -en "\033[1A" c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}" if [ "${avgdays}" != "noavg" ]; then if [ $(bc -l <<< "${avgdays} < 1") -eq 1 ]; then avgdays_disp="$(ROUND_NUMBER $(bc <<< "scale=2; ${avgdays}*24*60") 0) Minute" else avgdays_disp="${avgdays} Day" fi echo -en "Pulling data and calculating ${avgdays_disp} average ... " else echo -en "Pulling data ... " fi reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays}) echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}${idsCL[Default]}" else echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Sensor Offline${idsCL[Default]}" fi echo last_sensor=${SENSOR%%-*} fi fi done rm -f ${PW_TMPFOLDER}/.fanspeed.* end=`date +%s`; runtime=$((end-start)); echo -e "runtime: ${runtime}s" [ "${PW_ACTION}" != "" ] && echo -e "${idsCL[Default]}" } CHECKTEMPSENSOR(){ if [ "${PW_THRESHOLDS[${1}]}" != "" ]; then echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit fi if [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then GAUGESH="'%" elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then GAUGESH="'R" else GAUGESH="'F" fi if [ "${6}" != "noavg" ]; then [ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3 if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays}) else average=$(AVERAGETEMP ${PW_SENSOR_ID[${1}]} ${avgdays} ${5}) fi else average='' fi if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${1}]}" == "lacrosse" ]; then INFO=$(curl -m 7 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq) if [ "${INFO}" != "" ]; then if [[ "${1}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi else temp_c='' fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then if [[ "${1}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} elif [[ "${1}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp) if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then temp_c=${2} temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'` temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'` elif [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then if [ "${1}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi fi if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ "${1}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi # echo "HERE: '${temp_c}' -> '${temp_f}'" # echo -en "" c=0; spct=''; [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]] && spc1=`expr 5 - ${#temp_f}` || spc1=`expr 6 - ${#temp_f}` until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done if [ "${average}" != "" ]; then ([ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]) && average_disp=$(IDS_NUMBER_FORMAT ${average}) || average_disp=${average} c=0; spca=''; spc1=`expr 6 - ${#average_disp}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done if [ "${average_disp}" == "" ]; then averagedisp="${idsCL[Green]}" elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then averagedisp="${idsCL[Yellow]}${average}" elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then averagedisp="${idsCL[LightRed]}${average_disp}" else averagedisp="${idsCL[Green]}${average_disp}" fi fi if [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then temp_f_disp=$(IDS_NUMBER_FORMAT ${temp_f} 0) temp_warn_disp=$(IDS_NUMBER_FORMAT ${temp_warn} 0) temp_crit_disp=$(IDS_NUMBER_FORMAT ${temp_crit} 0) else temp_f_disp=${temp_f} temp_warn_disp=${temp_warn} temp_crit_disp=${temp_crit} fi if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then lclr="${idsCL[Yellow]}" lmd="WARNING " elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then lclr="${idsCL[LightRed]}" lmd="CRITICAL" else lclr="${idsCL[Green]}" lmd="Normal " fi echo -en "${lclr}${lmd}${spct}${temp_f_disp}${GAUGESH}" [ "${average}" != "" ] && echo -en " /${spca}${averagedisp}${GAUGESH}" if [[ "${1}" != *"FAN"* ]]; then tmp="${spct}${temp_f_disp}${GAUGESH}" c=0; spc=''; spc1=`expr 12 - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done [ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' ' [ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' ' echo -e "${idsCL[Default]}${spc}[${spcw}${idsCL[Yellow]}${temp_warn_disp}${GAUGESH}${idsCL[Default]} /${spcc}${idsCL[LightRed]}${temp_crit_disp}${GAUGESH}=>${idsCL[Default]} ]" fi else echo -e "${idsCL[Yellow]}Sensor Offline${idsCL[Default]}" fi } AVERAGETEMP(){ temptotal=0 daysback=${2} sensorid=${1} logsfound=0; if [ "${3}" != "" ]; then entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'")) entityid=${entityid[1]} WHERE="sensorid=${sensorid} AND entity=${entityid}" else WHERE="sensorid=${sensorid}" fi [ $(bc -l <<< "${daysback} < 1") -eq 1 ] && interval="$(ROUND $(bc <<< "scale=2; ${daysback}*24*60")) HOUR" || interval="${daysback} MINUTE" while IFS=$'\t' read date temp hum speed ;do if [ "${sensorid}" == "14" ] || [ "${sensorid}" == "19" ]; then temptotal=$(bc <<< "scale=0; ${temptotal}+${speed}") else temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}") fi ((logsfound++)) done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum,speed from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${interval}) AND NOW() ORDER BY id DESC;") if [ "${sensorid}" == "14" ] || [ "${sensorid}" == "19" ]; then # average=$(bc <<< "scale=0; ${temptotal}/${logsfound}") average=$(ROUND_NUMBER $(bc <<< "scale=2; ${temptotal}/${logsfound}") 0) else average=$(bc <<< "scale=2; ${temptotal}/${logsfound}") fi (( $(bc <<<"${average} < 1") )) && [[ "${average}" = *"."* ]] && average="0${average}" echo ${average} # echo ${logsfound} } CHECKTEMP_SERVICE(){ cw=20 runEvery=60 #SECONDS SCAN_THREADS=4 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Temp' Service Startup" >> ${PW_LOGFILE} SENDTEMPS "TEMP Service Startup" & [ ! -d ${PW_TMPFOLDER}/.lastinsert ] && mkdir ${PW_TMPFOLDER}/.lastinsert [ -d ${PW_TMPFOLDER}/.lasttemp ] && rm -Rf ${PW_TMPFOLDER}/.lasttemp mkdir ${PW_TMPFOLDER}/.lasttemp rm -f ${PW_TMPFOLDER}/ztemprun.* rm -f ${PW_TMPFOLDER}/.fancheck.* while true; do start=$(date +%s) source ${PW_FOLDER}/defaults.inc # time="$(date +%H)$(date +%M)" # if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then # if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then # touch ${PW_TMPFOLDER}/.sentdaily # SENDTEMPS "Daily Temp Readings" yes & # fi # fi SENSORS_CHECKED=() for SENSORa in ${PW_SENSOR_ORD[@]}; do until [ $(ls ${PW_TMPFOLDER}/ztemprun.*.running 2>/dev/null | wc -l) -lt ${SCAN_THREADS} ]; do tmp=tmp done CHECKTEMP_SERVICE_RUNSENSOR ${SENSORa} & SENSORS_CHECKED+=(${SENSORa}) done s_count=0; completed=false; scanstart=$(date +%s) until [ "${completed}" == "true" ]; do for sensor in ${PW_TMPFOLDER}/ztemprun.*.done; do if [ "$(grep -oP '(?<=ztemprun.).*?(?=.done)' <<< "${sensor}")" != "*" ]; then rm -f ${sensor} ((s_count++)) fi done [ ${s_count} -eq ${#SENSORS_CHECKED[@]} ] && completed=true if [ $(($(date +%s)-scanstart)) -gt 300 ]; then SENDNOTICE "PW Scan Stuck" "PW Temp scan has been running for 5mins, try breaking...." 1 break fi done end=`date +%s`; runtime=$((end-start)) # echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s" [ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s done # & } CHECKTEMP_SERVICE_RUNSENSOR(){ SENSORa=${1} logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa} touch ${PW_TMPFOLDER}/ztemprun.${SENSORa}.running [ ! -f ${logtemp} ] && touch ${logtemp} [ -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done ] && rm -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done [ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0 if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ] || ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then # if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then if [ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ]; then SENDNOTICE "${SENSORa} Data" "${SENSORa} is reporting data again" echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is reporting data again" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSORa}*-error.* rm -f ${PW_TMPFOLDER}/${SENSORa}.down elif [ $(ls ${PW_TMPFOLDER}/${SENSORa}*-error.* 2>/dev/null | wc -l) -gt 0 ]; then rm -f ${PW_TMPFOLDER}/${SENSORa}*-error.* rm -f ${PW_TMPFOLDER}/${SENSORa}.down fi rm -f ${PW_TMPFOLDER}/${SENSORa}.down if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}" unset IFS [ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off t=1 ####### Fan Issue Workaround ####### if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && ([ ! -f ${PW_TMPFOLDER}/.fancheck.${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.fancheck.${SENSORa})) -ge 300 ]); then # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "${vHOSTFAN}" != "" ]; then # FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'` 0) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $9}' | sed -e 's/%//g') else FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') fi else FanSpeed='' fi fi else SENSORS_CHECK=(${SENSORa}) fi if [ "${SENSORS_CHECK}" != "" ]; then for SENSOR in "${SENSORS_CHECK[@]}"; do if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit [ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit} fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then INFO=$(curl -m 5 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq) if [[ "${SENSOR}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity') elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then if [[ "${SENSOR}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' elif [[ "${SENSOR}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) temp_h='' if [ "${SENSOR}" == "Powerwall-FAN" ] && [ ${temp_c} -lt 1000 ]; then START_POWERWALL_FAN >/dev/null 2>&1 & fi fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g') temp_c=`echo "scale=1; ${temp_c}/10" | bc` temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g') temp_warn=`echo "scale=1; ${temp_warn}/10" | bc` temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc) temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g') temp_crit=`echo "scale=1; ${temp_crit}/10" | bc` temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc) temp_crit_sys=${temp_crit} temp_h='' vSENSOR=${SENSOR} SENSOR="${SENSORa}-${vSENSOR// /_}" ((t++)) elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi temp_h='' fi [ ! -f ${PW_TMPFOLDER}/.lasttemp/${SENSOR} ] && echo 0 > ${PW_TMPFOLDER}/.lasttemp/${SENSOR} last_temp=$(cat ${PW_TMPFOLDER}/.lasttemp/${SENSOR}) logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR} if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then [ -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.disable if [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then SENDNOTICE "${SENSOR} Online" "${SENSOR} Sensor is back online" echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is back online" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSOR}.down fi if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again" echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSOR}-error.* elif [ $(ls ${PW_TMPFOLDER}/${SENSOR}-error.* 2>/dev/null | wc -l) -gt 0 ]; then rm -f ${PW_TMPFOLDER}/${SENSOR}-error.* fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then GAUGE="CPU-USAGE" elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then GAUGE="RPM" else GAUGE="TEMP" fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then GAUGESH='%' elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then GAUGESH="'R" else GAUGESH="'F" fi #### [ -f ${PW_TMPFOLDER}/${SENSOR}.* ] && rm -f ${PW_TMPFOLDER}/${SENSOR}.* if [ "${SENSOR}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi # echo "HERE: [${SENSOR//-/}] - ${last_temp} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}" # [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND" if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys fi if [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]] && (( $(bc <<< "${temp_f} < ${temp_warn}") )); then if (( $(bc <<< "${last_temp} >= 1") )); then [ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH} Previous Temp: ${last_temp}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp}${GAUGESH}" >> ${logtemp} # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off elif [ "${last_temp}" == "0" ]; then # SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup # NORMAL TEMP: ${temp_f}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off elif [ "${relog}" == "1" ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp} fi ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && rm -f ${PW_TMPFOLDER}/temp.* rm -f ${PW_TMPFOLDER}/power.${SENSORa}.* if [ "${last_temp}" != "1" ]; then last_temp=1 echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi ####### Fan Issue Workaround ####### if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]; then if [ ! -f ${PW_TMPFOLDER}/.fancheck.${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.fancheck.${SENSORa})) -ge 300 ]; then if [ ${FanSpeed} -ne 30 ]; then if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] || [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.10.2.27" ] || [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.10.2.26" ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal 30%" ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x1E >/dev/null 2>&1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to 30%" >> ${PW_LOGFILE} elif [ ${FanSpeed} -ge 90 ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1 fi fi echo ${FanSpeed} >| ${PW_TMPFOLDER}/.fancheck.${SENSORa} fi fi elif [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]]; then if ([ ! "${last_temp}" ] || (( $(bc <<< "${last_temp} < 2") ))); then if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi fi temp_diff=`echo "scale=2; ${last_temp}-${temp_f}" | bc` [ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc` if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then # ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && touch ${PW_TMPFOLDER}/temp.warn touch ${PW_TMPFOLDER}/temp.warn.${SENSOR} if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp} fi ####### Fan Issue Workaround ####### if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]]; then if [ ! -f ${PW_TMPFOLDER}/.fancheck.${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.fancheck.${SENSORa})) -ge 300 ]; then # if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then if [ ${FanSpeed} -ne 80 ]; then if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] || [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.10.2.27" ] || [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.10.2.26" ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%" 1 ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE} else SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Fans not set at 80%" 1 fi fi echo ${FanSpeed} >| ${PW_TMPFOLDER}/.fancheck.${SENSORa} fi fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}${GAUGESH})" >> ${PW_LOGFILE} if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi # if ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && [ ! -f ${PW_TMPFOLDER}/temp.crit ]; then if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f ${PW_TMPFOLDER}/temp.crit.${SENSOR} ]; then touch ${PW_TMPFOLDER}/temp.crit.${SENSOR} SHUTDOWN_MAIN ${SENSOR} & echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp} # elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then # SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 # SHUTDOWN_SERVER ${SENSORa} -esxi off & # touch ${PW_TMPFOLDER}/power.${SENSORa}.off fi fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}${GAUGESH})" >> ${PW_LOGFILE} if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi # if ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && [ ! -f ${PW_TMPFOLDER}/temp.critsys ]; then if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f ${PW_TMPFOLDER}/temp.critsys.${SENSOR} ]; then touch ${PW_TMPFOLDER}/temp.critsys.${SENSOR} SHUTDOWN_SYS ${SENSOR} TEMP & SHUTDOWN_CRIT ${SENSOR} & echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} fi fi fi fi [ -f ${PW_TMPFOLDER}/.lastinsert/${SENSOR} ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastinsert/${SENSOR})) || lastinsert=100000 if [ $(ls ${PW_TMPFOLDER}/temp.* 2>/dev/null | wc -l) -gt 0 ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*.low")" != "" ]; then wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc) else wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc) fi if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) if [ "${entityid[1]}" == "" ]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')" entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) fi entityid=${entityid[1]} QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${vHOSTFAN}" != "" ]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('14','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')" fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" elif [ "${temp_h}" != "" ]; then QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')" else QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" fi ${mysql_conn} -e "${QRY}" # if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then # ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')" # fi touch ${PW_TMPFOLDER}/.lastinsert/${SENSOR} fi #### LOGGING OF SERVERROOM SENSOR DIFFERENCE if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then if [ "${temp_f}" != "" ] && (( $(bc <<< "${temp_f} >= 2") )) && [ -f ${PW_TMPFOLDER}/.serverroomth ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.serverroomth)) -lt 30 ]; then [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference temp_difference=$(bc <<< "scale=2; $(cat ${PW_TMPFOLDER}/.serverroomth)-${temp_f}") # (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc` echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference # echo "[$(date +'%Y-%m-%d %H:%M:%S')] ${temp_f} - $(cat ${PW_TMPFOLDER}/.serverroomth) = ${temp_difference}" rm -f ${PW_TMPFOLDER}/.serverroomth fi fi elif [ "${SENSOR}" == "ServerRoomTH" ]; then echo ${temp_f} >| ${PW_TMPFOLDER}/.serverroomth fi ################### else [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then touch ${PW_TMPFOLDER}/${SENSOR}-error.reading elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then touch ${PW_TMPFOLDER}/${SENSOR}-error.sent SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} fi fi done fi else [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then touch ${PW_TMPFOLDER}/${SENSORa}-error.reading elif [ ! -f ${PW_TMPFOLDER}/${SENSORa}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then touch ${PW_TMPFOLDER}/${SENSORa}-error.sent SENDNOTICE "${SENSORa} Sensor ERROR" "ERROR reading sensor data" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp} fi fi rm -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.running touch ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done } SENDTEMPS(){ st_temps=$(SENDTEMP ${2}) SENDNOTICE "${1}" "${st_temps}" } SENDTEMP(){ echo echo -en "Sensor Temperature(s)" [ "${1}" != "" ] && echo " Now / 1DayAVG / 7DayAVG" || echo DIVIDER . . 40 for SENSORa in ${PW_SENSOR_ORD[@]}; do ([[ "${SENSORa}" = *"-Probe"* ]] || [[ "${SENSORa}" = *"-Usage"* ]]) && cw=25 || cw=30 if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ] && ([ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ] || ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ])); then SENSORS_CHECK=(${SENSORa}) for SENSOR in "${SENSORS_CHECK[@]}"; do if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then GAUGESH="'%" elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then GAUGESH="'R" else GAUGESH="'F" fi [ "${last_sensor}" != "" ] && [ "${last_sensor}" != "${SENSOR%%-*}" ] && echo if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${SENSORa}/SENSOR -C 1` echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ]; then INFO=$(curl -m 5 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq) if [[ "${SENSORa}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then if [[ "${SENSORa}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} elif [[ "${SENSORa}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} vcgencmd measure_temp) if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*} temp_c=${temp_c#*=} fi # elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then # temp_c=${2} # temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'` # temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'` elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi else temp_c='' fi if [ "${1}" != "" ]; then if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_}) average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_}) # average7=0 else average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1) average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7) # average7=0 fi fi c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ "${SENSOR}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f_disp=$(IDS_NUMBER_FORMAT ${temp_f}) if [ "${1}" != "" ]; then average1_disp=$(IDS_NUMBER_FORMAT ${average1}) average7_disp=$(IDS_NUMBER_FORMAT ${average7}) fi else temp_f_disp=${temp_f} if [ "${1}" != "" ]; then average1_disp=${average1} average7_disp=${average7} fi fi [ "${1}" != "" ] && echo -e "${SENSOR}${spc}${temp_f_disp}${GAUGESH} / ${average1_disp}${GAUGESH} / ${average7_disp}${GAUGESH}" || echo -e "${SENSOR}${spc}${temp_f_disp}${GAUGESH}" else echo -e "${SENSOR}${spc}No Data" fi last_sensor=${SENSORa%%-*} done fi done } GET_LACROSSE_INFO(){ INFO=$(/bin/python3 ${PW_INCFOLDER}/getInfo.lacrosse.py "${PW_LACROSSE_ID[${1}]}" | sed -e "s/\[{'//g" | sed -e "s/}\]//g" | sed -e "s/', '/~/g" | sed -e "s/': '/:/g" | sed -e "s/': /:/g") IFS=$'~'; read -rd '' -a INFO <<<"${INFO}"; unset IFS; declare -A DEVICE_INFO for DINFOtmp in "${INFO[@]}"; do key=${DINFOtmp%%:*}; val=${DINFOtmp##*:}; val=${val//[$'\t\r\n ']}; DEVICE_INFO[${key}]=${val}; done echo ${DEVICE_INFO['probe_temp']} } SERVERROOM_TEMP_DIFFERENCE(){ start=$(date +%s) scandefault=288 while [ $# -gt 0 ]; do case "${1}" in -l|-last) LAST=${2};; -h|-help|--help) echo -e "Usage: ${idsCL[Yellow]}[powerwall or pw] tempdiff {opt}${idsCL[Default]} {" width=15 printf "%-${width}s- %s\n" " -l|-last" "(number of last scan times to average from, defaults to ${scandefault})" echo -e "}\n" exit 0;; esac shift done [ "${LAST}" == "" ] && LAST=${scandefault} echo -en "${idsCL[Yellow]}Pulling data and calculating average temperature differences for the last '${LAST}' scans ... " last_tempdiff=$(tail -n 1 ${PW_TMPFOLDER}/.log.sr.difference) last_tempdiff=${last_tempdiff#*~} # if [ $(bc -l <<< "${last_tempdiff} < 1") -eq 1 ]; then [ "${last_tempdiff}" = "0" ] && last_tempdiff="0.0" || last_tempdiff="0${last_tempdiff}"; fi last_tempdiff=`IDS_NUMBER_FORMAT ${last_tempdiff} 2 yes` tempdiffs=$(tail -n ${LAST} ${PW_TMPFOLDER}/.log.sr.difference) largest_tempdiff=0 IFS=$'\n' read -rd '' -a tempdiffs <<<"${tempdiffs}" unset IFS tempdiff_total=0; scansfound=0 tempdiff_large_total=0; scansfound_large=0 for tempdiff in "${tempdiffs[@]}"; do [[ "${tempdiff}" = *"~"* ]] && tempdiff=${tempdiff#*~} if [ $(bc -l <<< "${tempdiff} <= 15") -eq 1 ]; then # (( $(bc <<<"${tempdiff} < 0") )) && tempdiff_tmp1=`echo "scale=2; ${tempdiff} * -1" | bc` || tempdiff_tmp1=${tempdiff} # (( $(bc <<<"${largest_tempdiff} < 0") )) && tempdiff_tmp2=`echo "scale=2; ${largest_tempdiff} * -1" | bc` || tempdiff_tmp2=${largest_tempdiff} # [ $(bc -l <<< "${tempdiff_tmp1} >= ${tempdiff_tmp2}") -eq 1 ] && largest_tempdiff=${tempdiff} # (( $(bc <<<"${tempdiff} < 0") )) && tempdiff_tmp1=`echo "scale=2; ${tempdiff} * -1" | bc` || tempdiff_tmp1=${tempdiff} # [ $(bc -l <<< "${tempdiff_tmp1} >= ${largest_tempdiff}") -eq 1 ] && largest_tempdiff=${tempdiff_tmp1} # [ $(bc -l <<< "${tempdiff} >= ${largest_tempdiff}") -eq 1 ] && largest_tempdiff=${tempdiff} tempdiff_total=$(bc <<< "scale=2; ${tempdiff_total}+${tempdiff}") ((scansfound++)) else tempdiff_large_total=$(bc <<< "scale=2; ${tempdiff_large_total}+${tempdiff}") ((scansfound_large++)) fi done # largest_tempdiff=`IDS_NUMBER_FORMAT ${largest_tempdiff} 2 yes` [ ${scansfound} -eq 0 ] && average=0 || average=$(bc <<< "scale=2; ${tempdiff_total}/${scansfound}") [ ${scansfound_large} -eq 0 ] && average_large=0 || average_large=$(bc <<< "scale=2; ${tempdiff_large_total}/${scansfound_large}") # if [ $(bc -l <<< "${average} < 1") -eq 1 ]; then [ "${average}" = "0" ] && average="0.0" || average="0${average}"; fi average=`IDS_NUMBER_FORMAT ${average} 2 yes` [ -f ${PW_TMPFOLDER}/.log.sr.difference ] && last_scancheck=$(date -d @$(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference) "+%Y-%m-%d %H:%M:%S") || last_scancheck="" echo -en "\r\033[K" echo -e "${idsCL[LightCyan]}Scan Type Scans Average Temp Difference${idsCL[Default]}" DIVIDER . lightCyan 60 echo -e "${idsCL[Green]}Average for last `IDS_NUMBER_FORMAT ${scansfound}` ${idsCL[LightGreen]}${average}'F${idsCL[Default]}" DIVIDER . lightCyan 60 echo -e "${idsCL[Cyan]} - Large runs `IDS_NUMBER_FORMAT ${scansfound_large}` ${idsCL[LightCyan]}${average_large}'F${idsCL[Default]}" echo # echo -e "${idsCL[LightCyan]}Largest Temp Difference : ${idsCL[LightGreen]}${largest_tempdiff}'F${idsCL[Default]}" # echo echo -e "${idsCL[LightCyan]}Last Temp Difference : ${idsCL[LightGreen]}${last_tempdiff}'F${idsCL[Default]}" echo -e "${idsCL[LightCyan]}Last Check : ${idsCL[LightGreen]}${last_scancheck}${idsCL[Default]}" echo echo -e "${idsCL[LightCyan]}Current Temp Deviation : ${idsCL[LightRed]}${PW_ServerRoomTH_Dev}'F${idsCL[Default]}" if [ ${scansfound} -le 500 ]; then if [ ${scansfound} -gt 15 ]; then echo -en "\n${idsCL[LightCyan]}List out these '${scansfound}' scan times? (y/N): ${idsCL[Default]}" read -n1 choice if [ "${choice^^}" == "Y" ]; then echo -en "\r\033[K" gofor=1 else gofor=0 fi else gofor=1 echo fi if [ ${gofor} -eq 1 ]; then lastscansnum=${scansfound} last_tempdiffs=$(tail -n ${lastscansnum} ${PW_TMPFOLDER}/.log.sr.difference) IFS=$'\n' read -rd '' -a last_tempdiffs <<<"${last_tempdiffs}" unset IFS echo -e "${idsCL[LightCyan]}Last ${lastscansnum} Scan Times\n${idsCL[Green]}------------------------------------------${idsCL[Default]}" for tempdiff in "${last_tempdiffs[@]}"; do if [[ "${tempdiff}" = *"~"* ]]; then dt="${tempdiff%~*}" tempdiff=${tempdiff#*~} # if [ $(bc -l <<< "${tempdiff} < 1") -eq 1 ]; then [ "${tempdiff}" = "0" ] && tempdiff="0.0" || tempdiff="0${tempdiff}"; fi tempdiff=`IDS_NUMBER_FORMAT ${tempdiff} 2` else dt=" " fi echo -e "${idsCL[White]} ${dt} : ${idsCL[Yellow]}${tempdiff}'F${idsCL[Default]}" done fi fi end=`date +%s`; runtime=$((end-start)) echo -e "\nruntime: ${runtime}s\n" }