GET_HOST_FANSPEED(){ if [ -f ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]} ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]})) -lt 600 ] && ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]}.set ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]})) -lt $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]}.set)) ]); then FanSpeed=$(cat ${PW_TMPFOLDER}/.lastfancheck/${PW_ESXI_HOST_NAMES[${1}]}) else vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${1}]}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $9}' | sed -e 's/%//g') else FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') fi fi echo ${FanSpeed} > ${PW_TMPFOLDER}/.fanspeed.${PW_ESXI_HOST_NAMES[${1}]}.${2} } START_POWERWALL_FAN(){ pwmconfig & sleep 4s killall pwmconfig temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) if [ ${temp_c} -gt 0 ]; then SENDNOTICE "Starting up the ${SENSOR}" "Startup confirmed, the fan is now reading: ${temp_c}" else SENDNOTICE "FAILED to start ${SENSOR}" "Cannot start the fan, ERROR WILL ROBINSON!!!" fi } CHECKTEMP(){ start=`date +%s` cw=25 cdt=$(date +%Y-%m-%d-%H-%M-%S) while [ $# -gt 0 ]; do case "$1" in -a|-avg) avgdays=${2};; -S|-sensors) temptype="sensors";; -e|-esxi) temptype="esxi";; -f|-fans) temptype="fans";; -s|-search) search=${2};; esac shift 1 done if [ "${temptype}" != "sensors" ]; then for ESXIHOST in "${PW_ALLHOSTS[@]}"; do if [ ! -f ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.down ] && ([ "${search}" == "" ] || [[ "${PW_ESXI_HOST_NAMES[${ESXIHOST}],,}" = *"${search,,}"* ]]); then GET_HOST_FANSPEED ${ESXIHOST} ${cdt} & fi done [ "${PW_ESXI_DEF_FANSPEED}" != "" ] && ESXI_DEF_FANSPEED=${PW_ESXI_DEF_FANSPEED} || ESXI_DEF_FANSPEED=30 [ "${PW_ESXI_DEF_EXHAUST_MIN}" != "" ] && ESXI_DEFAULT_EXHAUST_MIN=${PW_ESXI_DEF_EXHAUST_MIN} || ESXI_DEFAULT_EXHAUST_MIN=85 [ "${PW_ESXI_FANSPEED_GL_ZONE}" != "" ] && ESXI_FANSPEED_GL_ZONE=${PW_ESXI_FANSPEED_GL_ZONE} || ESXI_FANSPEED_GL_ZONE=2.5 GL_HIGH=$(bc <<< "scale=2; ${ESXI_DEFAULT_EXHAUST_MIN}+${ESXI_FANSPEED_GL_ZONE}") GL_LOW=$(bc <<< "scale=2; ${ESXI_DEFAULT_EXHAUST_MIN}-${ESXI_FANSPEED_GL_ZONE}") fi echo if [ "${avgdays}" != "" ]; then if [[ "${avgdays^^}" = *"M"* ]]; then avgdays_disp="${avgdays::-1}Min" elif [[ "${avgdays^^}" = *"H"* ]]; then avgdays_disp="${avgdays::-1}Hr" elif [[ "${avgdays^^}" = *"D"* ]]; then avgdays_disp="${avgdays::-1}Day" elif [[ "${avgdays}" =~ ^[0-9]+$ ]]; then avgdays_disp="${avgdays}Day" fi else avgdays=noavg fi if [ "${temptype}" != "esxi" ]; then DIVIDER . yellow ${PRI_WIDTH} if [ "${avgdays}" != "noavg" ]; then echo -e "${idsCL[LightYellow]}SENSOR TEMPERATURE(S) Current / ${idsCL[LightYellow]}${avgdays_disp}AVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" else echo -e "${idsCL[LightYellow]}SENSOR TEMPERATURE(S) Current ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" fi DIVIDER . yellow ${PRI_WIDTH} for SENSOR in ${PW_SENSOR_ORD[@]}; do if ([ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]) && [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ] && ([ "${temptype}" != "fans" ] || ([ "${temptype}" == "fans" ] && [[ "${SENSOR}" = *"Room"* ]])); then [ "${last_sensor}" == "${SENSOR%%-*}" ] && echo -en "\033[1A" c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}" if [ "${avgdays}" != "noavg" ]; then if [[ "${avgdays^^}" = *"M"* ]]; then avgdays_disp="${avgdays::-1} Minute" elif [[ "${avgdays^^}" = *"H"* ]]; then avgdays_disp="${avgdays::-1} Hour" elif [[ "${avgdays^^}" = *"D"* ]]; then avgdays_disp="${avgdays::-1} Day" elif [[ "${avgdays}" =~ ^[0-9]+$ ]]; then avgdays_disp="${avgdays::-1} Day" fi echo -en "Pulling data and calculating ${avgdays_disp} average ... " else echo -en "Pulling data ... " fi reading=$(CHECKTEMPSENSOR ${SENSOR} . . . . ${avgdays}) echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${reading}${idsCL[Default]}" else echo -e "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR}${idsST[Reset]}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Sensor Offline${idsCL[Default]}" fi echo last_sensor=${SENSOR%%-*} fi done DIVIDER . yellow ${PRI_WIDTH} fi if [ "${temptype}" != "sensors" ]; then DIVIDER . yellow ${PRI_WIDTH} if [ "${avgdays}" != "noavg" ]; then echo -e "${idsCL[LightYellow]}ESXI HOST TEMPERATURE(S) Current / ${idsCL[LightYellow]}${avgdays_disp}AVG ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" else echo -e "${idsCL[LightYellow]}ESXI HOST TEMPERATURE(S) Current ${idsCL[Yellow]}warning${idsCL[Default]} / ${idsCL[LightRed]}critical${idsCL[Default]}" fi DIVIDER . yellow ${PRI_WIDTH} for SENSOR in ${PW_SENSOR_ORD[@]}; do if ([ "${search}" == "" ] || [[ "${SENSOR,,}" = *"${search,,}"* ]]) && [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "esxi" ]; then vHOSTiDRACIP=${PW_HOST_IDRACS[${SENSOR}]} vHOSTIP=${PW_HOST_ADDRESSES[${SENSOR}]} vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTiDRACIP}]} ([ "${temptype}" == "fans" ] && [ "${SENSOR}" == "iDS-vMS-Host6" ]) && echo -e "${idsCL[LightYellow]}${SENSOR} PSU is not compatible, Fans are instead synced with 'iDS-vMS-Host7'${idsCL[Default]}" echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]" if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then if [ "${avgdays}" != "noavg" ]; then echo -en "${idsCL[LightCyan]} Getting FanSpeed and calculating ${avgdays_disp} average..." else echo -en "${idsCL[LightCyan]} Getting FanSpeed..." fi [ ! -f ${PW_TMPFOLDER}/.fanspeed.${SENSOR}.${cdt} ] && until [ -f ${PW_TMPFOLDER}/.fanspeed.${SENSOR}.${cdt} ]; do tmp=tmp; done RPM=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSOR}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) [ "${avgdays}" != "noavg" ] && fan_average=$(bc <<< "scale=0; $(ROUNDHALVES $(bc <<< "scale=3; $(AVERAGETEMP ${SENSOR} ${avgdays} FanSpeed)/10"))*10/1") echo -en "\r\033[K${idsCL[White]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[White]}[${vHOSTiDRACIP}]" [ "${SENSOR}" == "iDS-vMS-Host6" ] && cSENSOR="iDS-vMS-Host7" || cSENSOR=${SENSOR} if [ -f ${PW_TMPFOLDER}/.lastfancheck/${cSENSOR}.gl ]; then FSC="${idsCL[LightGreen]}${idsST[Bold]}" elif [ -f ${PW_TMPFOLDER}/.lastfancheck/${cSENSOR}.set ]; then FSC="${idsCL[LightGreen]}" else FSC="${idsCL[LightYellow]}" fi echo -en "${idsCL[LightCyan]} [FanSpeed=${FSC}$(cat ${PW_TMPFOLDER}/.fanspeed.${SENSOR}.${cdt})${idsST[Reset]}${idsCL[LightCyan]}'% " if [ "${avgdays}" != "noavg" ]; then echo -e "${idsCL[LightYellow]} AVG=${fan_average}'%${idsCL[LightCyan]}]${idsCL[Default]}" else echo -e "($(IDS_NUMBER_FORMAT ${RPM})'RPM)]${idsCL[Default]}" fi else echo -e "${idsCL[Default]}" fi # if [ "${temptype}" != "fans" ]; then DIVIDER . lightCyan ${PRI_WIDTH} echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ... " # if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') IFS=$'\n' read -rd '' -a vHOST_TEMPNAMES <<<"${vHOST_TEMPNAMES}" unset IFS echo -en "\r\033[K" if [ ${#vHOST_TEMPNAMES[@]} -gt 1 ]; then t=1 for sensorname in "${vHOST_TEMPNAMES[@]}"; do if [ "${temptype}" != "fans" ] || ([ "${temptype}" == "fans" ] && [[ "${sensorname}" = *"Exhaust"* ]]); then temp_c=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g') temp_c=`echo "scale=1; ${temp_c}/10" | bc` temp_warn=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g') temp_warn=`echo "scale=1; ${temp_warn}/10" | bc` temp_crit=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g') temp_crit=`echo "scale=1; ${temp_crit}/10" | bc` c=0; spc=''; spc1=`expr ${cw} - ${#sensorname} - 1`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done [ "${avgdays}" != "noavg" ] && echo -en "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${idsCL[Yellow]}Calculating ${avgdays_disp} average ... " reading=$(CHECKTEMPSENSOR ${SENSOR} ${temp_c} ${temp_warn} ${temp_crit} ${sensorname// /_} ${avgdays} ${temptype}) [ "${avgdays}" != "noavg" ] && echo -en "\r\033[K" echo -e "${idsCL[LightCyan]} ${sensorname}${spc}${idsCL[Default]}: ${reading}" if [ "${temptype}" == "fans" ]; then if [ -f ${PW_TMPFOLDER}/.lastfancheck/${cSENSOR}.gl ]; then FSC="${idsCL[LightGreen]}" elif [ -f ${PW_TMPFOLDER}/.lastfancheck/${cSENSOR}.set ]; then FSC="${idsCL[Green]}" else FSC="${idsCL[LightYellow]}" fi echo -e " Golidlock Zone : ${idsCL[LightGreen]}${ESXI_DEFAULT_EXHAUST_MIN}°F (±${ESXI_FANSPEED_GL_ZONE}°F) ${idsCL[Default]}[ ${idsCL[Green]}${GL_LOW}°F${idsCL[Default]} <--> ${idsCL[LightYellow]}${GL_HIGH}°F${idsCL[Default]} ]" echo -e " Last checked : ${idsCL[LightCyan]}$(date -r ${PW_TMPFOLDER}/.lastfancheck/${SENSOR} "+%m/%d @ %l:%M%p")${idsCL[Default]}" echo -e " Last set : ${FSC}$(date -r ${PW_TMPFOLDER}/.lastfancheck/${SENSOR}.set "+%m/%d @ %l:%M%p")${idsCL[Default]}" fi fi ((t++)) done else echo -e "${idsCL[LightCyan]} ${vHOSTNAME}${spc}${idsCL[Default]}: No Data" fi else #host down echo -e "\r\033[K${idsCL[LightRed]}${SENSOR} Host is down${idsCL[Default]}" # if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then # SENDNOTICE "${vHOSTiDRACIP} ERROR" "${vHOSTNAME} Host is down" 1 # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${vHOSTiDRACIP} - ${vHOSTNAME} Host is down" >> ${logtemp} # touch ${PW_TMPFOLDER}/${EHOST}.down # fi fi # fi ([ "${temptype}" != "fans" ] || ([ "${temptype}" == "fans" ] && [ "${SENSOR}" != "iDS-vMS-Host6" ])) && echo fi done fi rm -f ${PW_TMPFOLDER}/.fanspeed.* end=`date +%s`; runtime=$((end-start)); echo -e "runtime: ${runtime}s" [ "${PW_ACTION}" != "" ] && echo -e "${idsCL[Default]}" } CHECKTEMPSENSOR(){ if [ "${PW_THRESHOLDS[${1}]}" != "" ]; then echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit fi if [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then GAUGESH="'%" elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then GAUGESH="'R" else GAUGESH="°F" fi if [ "${6}" != "noavg" ]; then [ "${6}" != "" ] && [[ "${6}" =~ ^[0-9]+$ ]] && avgdays=${6} || avgdays=3 if [ "${PW_SENSOR_TYPE[${1}]}" != "esxi" ]; then average=$(AVERAGETEMP ${1} ${avgdays}) else average=$(AVERAGETEMP ${1} ${avgdays} ${5}) fi else average='' fi TIMEOUT=0 if [ "${PW_SENSOR_TYPE[${1}]}" == "mqtt" ]; then if ! mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1` ; then TIMEOUT=1; fi echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${1}]}" == "lacrosse" ]; then if ! INFO=$(curl -m 7 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php) ; then TIMEOUT=1; fi if [ "${INFO}" != "" ]; then if [[ "${1}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi else temp_c='' fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ]; then if [[ "${1}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} elif [[ "${1}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then if ! temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp) ; then TIMEOUT=1; fi if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then temp_c=${2} temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'` temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'` elif [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then if [ "${1}" == "Powerwall-CPU-Usage" ]; then if ! temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') ; then TIMEOUT=1 fi else if ! temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage") ; then TIMEOUT=1 fi fi fi if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ "${1}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${1}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi # echo "HERE: '${temp_c}' -> '${temp_f}'" # echo -en "" c=0; spct=''; [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]] && spc1=`expr 5 - ${#temp_f}` || spc1=`expr 6 - ${#temp_f}` until [ ${c} = ${spc1} ]; do spct="${spct} "; c=`expr ${c} + 1`; done if [ "${average}" != "" ]; then ([ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]) && average_disp=$(IDS_NUMBER_FORMAT ${average}) || average_disp=${average} c=0; spca=''; spc1=`expr 6 - ${#average_disp}`; until [ ${c} = ${spc1} ]; do spca="${spca} "; c=`expr ${c} + 1`; done if [ "${average_disp}" == "" ]; then averagedisp="${idsCL[Green]}" elif [ $(bc -l <<< "${average} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${average} < ${temp_crit}") -eq 1 ]; then averagedisp="${idsCL[Yellow]}${average}" elif [ $(bc -l <<< "${average} >= ${temp_crit}") -eq 1 ]; then averagedisp="${idsCL[LightRed]}${average_disp}" else averagedisp="${idsCL[Green]}${average_disp}" fi fi if [ "${PW_SENSOR_TYPE[${1}]}" == "system" ] && [[ "${1}" = *"FAN"* ]]; then temp_f_disp=$(IDS_NUMBER_FORMAT ${temp_f} 0) temp_warn_disp=$(IDS_NUMBER_FORMAT ${temp_warn} 0) temp_crit_disp=$(IDS_NUMBER_FORMAT ${temp_crit} 0) else temp_f_disp=${temp_f} temp_warn_disp=${temp_warn} temp_crit_disp=${temp_crit} fi if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then lclr="${idsCL[Yellow]}" lmd="WARNING " elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ]; then lclr="${idsCL[LightRed]}" lmd="CRITICAL" else lclr="${idsCL[Green]}" lmd="Normal " fi echo -en "${lclr}${lmd}${spct}${temp_f_disp}${GAUGESH}" [ "${average}" != "" ] && echo -en " /${spca}${averagedisp}${GAUGESH}" # if [ "${7}" == "fans" ]; then # echo -en "${idsCL[Default]}${spc}[${idsCL[LightGreen]}${GL_LOW}${GAUGESH}${idsCL[Default]}/${idsCL[Green]}${ESXI_DEFAULT_EXHAUST_MIN}${GAUGESH}${idsCL[Default]}/${idsCL[LightYellow]}${GL_HIGH}${GAUGESH}${idsCL[Default]}]" # # el if [[ "${1}" != *"FAN"* ]]; then tmp="${spct}${temp_f_disp}${GAUGESH}" [ "${average}" != "" ] && fws=12 || fws=21 c=0; spc=''; spc1=`expr ${fws} - ${#tmp}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done [ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' ' [ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' ' echo -en "${idsCL[Default]}${spc}[${spcw}${idsCL[Yellow]}${temp_warn_disp}${GAUGESH}${idsCL[Default]} /${spcc}${idsCL[LightRed]}${temp_crit_disp}${GAUGESH}=>${idsCL[Default]} ]" [ "${PW_SENSOR_TYPE[${1}]}" == "lacrosse" ] && [[ "${1}" != *"-Probe" ]] && echo -e " - $(echo ${INFO} | jq -r '.device0 .obs[].timestamp')" || echo fi elif [ ${TIMEOUT} -eq 1 ]; then echo -e "${idsCL[Yellow]}Sensor Timeout${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Sensor Offline${idsCL[Default]}" fi } AVERAGETEMP(){ temptotal=0 daysback=${2} sensor=${1} sensorid=${PW_SENSOR_ID[${sensor}]} logsfound=0; if [ "${3}" != "" ]; then entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${3}' AND sensorid='${sensorid}'")) entityid=${entityid[1]} WHERE="sensorid=${sensorid} AND entity=${entityid}" else WHERE="sensorid=${sensorid}" fi if [[ "${daysback^^}" = *"M"* ]]; then interval="${avgdays::-1} MINUTE" elif [[ "${daysback^^}" = *"H"* ]]; then interval="${avgdays::-1} HOUR" elif [[ "${daysback^^}" = *"D"* ]]; then interval="${avgdays::-1} DAY" elif [[ "${daysback}" =~ ^[0-9]+$ ]]; then interval="${avgdays} DAY" fi while IFS=$'\t' read date temp hum speed ;do if [ "${3}" == "FanSpeed" ] && [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${sensor}]}"* ]]; then temptotal=$(bc <<< "scale=2; ${temptotal}+${speed}") elif [ "${sensorid}" == "19" ]; then temptotal=$(bc <<< "scale=0; ${temptotal}+${speed}") else temptotal=$(bc <<< "scale=2; ${temptotal}+${temp}") fi ((logsfound++)) done < <(${mysql_conn} -se "USE servermonitor; SELECT date,temp,hum,speed from sensor_data WHERE ${WHERE} AND date BETWEEN DATE_SUB(DATE(NOW()), INTERVAL ${interval}) AND NOW() ORDER BY id DESC;") if [ "${3}" == "FanSpeed" ] && [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${sensor}]}"* ]]; then average=$(ROUND_NUMBER $(bc <<< "scale=2; ${temptotal}/${logsfound}") 2) elif [ "${sensorid}" == "19" ]; then # average=$(bc <<< "scale=0; ${temptotal}/${logsfound}") average=$(ROUND_NUMBER $(bc <<< "scale=2; ${temptotal}/${logsfound}") 0) else average=$(bc <<< "scale=2; ${temptotal}/${logsfound}") fi (( $(bc <<<"${average} < 1") )) && [[ "${average}" = *"."* ]] && average="0${average}" echo ${average} # echo ${logsfound} } CHECKTEMP_SERVICE(){ cw=20 runEvery=60 #SECONDS SCAN_THREADS=4 SENDTEMPS "TEMP Service Startup" & [ ! -d ${PW_TMPFOLDER}/.lastfancheck ] && mkdir ${PW_TMPFOLDER}/.lastfancheck #|| rm -f ${PW_TMPFOLDER}/.lastfancheck/*.set [ ! -d ${PW_TMPFOLDER}/.lastinsert ] && mkdir ${PW_TMPFOLDER}/.lastinsert [ -d ${PW_TMPFOLDER}/.lasttemp ] && rm -Rf ${PW_TMPFOLDER}/.lasttemp mkdir ${PW_TMPFOLDER}/.lasttemp rm -f ${PW_TMPFOLDER}/ztemprun.* while true; do start=$(date +%s) # time="$(date +%H)$(date +%M)" # if [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ]; then # if [ ! -f ${PW_TMPFOLDER}/.sentdaily ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.sentdaily)) -gt 600 ]; then # touch ${PW_TMPFOLDER}/.sentdaily # SENDTEMPS "Daily Temp Readings" yes & # fi # fi SENSORS_CHECKED=() for SENSORa in ${PW_SENSOR_ORD[@]}; do until [ $(ls ${PW_TMPFOLDER}/ztemprun.*.running 2>/dev/null | wc -l) -lt ${SCAN_THREADS} ]; do tmp=tmp done CHECKTEMP_SERVICE_RUNSENSOR ${SENSORa} & SENSORS_CHECKED+=(${SENSORa}) done s_count=0; completed=false; scanstart=$(date +%s) until [ "${completed}" == "true" ]; do for sensor in ${PW_TMPFOLDER}/ztemprun.*.done; do tsensor=$(grep -oP '(?<=ztemprun.).*?(?=.done)' <<< "${sensor}") if [ "${tsensor}" != "*" ]; then rm -f ${sensor} ((s_count++)) if [ -f ${PW_TMPFOLDER}/${tsensor}.down ] && [ -f ${PW_TMPFOLDER}/.lastfancheck/${tsensor} ]; then rm -f ${PW_TMPFOLDER}/.lastfancheck/${tsensor}* fi fi done [ ${s_count} -eq ${#SENSORS_CHECKED[@]} ] && completed=true if [ $(($(date +%s)-scanstart)) -gt 300 ]; then SENDNOTICE "PW Scan Stuck" "PW Temp scan has been running for 5mins, try breaking...." 1 break fi done end=`date +%s`; runtime=$((end-start)) # echo -e "runtime: ${runtime}s"; [ ${runtime} -lt ${runEvery} ] && echo "Sleeping for $(expr ${runEvery} - ${runtime})s" [ ${runtime} -lt ${runEvery} ] && sleep $(expr ${runEvery} - ${runtime})s done # & } CHECKTEMP_SERVICE_RUNSENSOR(){ SENSORa=${1} logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa} touch ${PW_TMPFOLDER}/ztemprun.${SENSORa}.running source ${PW_FOLDER}/defaults.inc [ "${PW_ESXI_DEF_FANSPEED}" != "" ] && ESXI_DEF_FANSPEED=${PW_ESXI_DEF_FANSPEED} || ESXI_DEF_FANSPEED=30 [ "${PW_ESXI_DEF_EXHAUST_MIN}" != "" ] && ESXI_DEFAULT_EXHAUST_MIN=${PW_ESXI_DEF_EXHAUST_MIN} || ESXI_DEFAULT_EXHAUST_MIN=85 [ "${PW_ESXI_FANSPEED_GL_ZONE}" != "" ] && ESXI_FANSPEED_GL_ZONE=${PW_ESXI_FANSPEED_GL_ZONE} || ESXI_FANSPEED_GL_ZONE=2.5 GL_HIGH=$(bc <<< "scale=2; ${ESXI_DEFAULT_EXHAUST_MIN}+${ESXI_FANSPEED_GL_ZONE}") GL_LOW=$(bc <<< "scale=2; ${ESXI_DEFAULT_EXHAUST_MIN}-${ESXI_FANSPEED_GL_ZONE}") [ ! -f ${logtemp} ] && touch ${logtemp} [ -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done ] && rm -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done [ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0 if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ] || ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]) || ([ "${PW_HOST_ADDRESSES[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_HOST_ADDRESSES[${SENSORa}]})" != "false" ]); then # if ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ]); then if [ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ]; then SENDNOTICE "${SENSORa} Data" "${SENSORa} is reporting data again" echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is reporting data again" >> ${logtemp} fi rm -f ${PW_TMPFOLDER}/${SENSORa}* if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}" unset IFS [ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off t=1 ####### Grab Fan Speeds ####### if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]); then # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "${vHOSTFAN}" != "" ]; then # FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'` 0) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $9}' | sed -e 's/%//g') else FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') fi echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} else FanSpeed='' fi else FanSpeed='' fi else SENSORS_CHECK=(${SENSORa}) fi if [ "${SENSORS_CHECK}" != "" ]; then for SENSOR in "${SENSORS_CHECK[@]}"; do if [ "${PW_THRESHOLDS[${SENSORa}]}" != "" ]; then echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 2 | read temp_crit [ "$(echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3)" != "" ] && echo ${PW_THRESHOLDS[${SENSORa}]} | cut -d',' -f 3 | read temp_crit_sys || temp_crit_sys=${temp_crit} fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` echo ${mqtt_message} | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then INFO=$(curl -m 5 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq) if [[ "${SENSOR}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi temp_h=$(echo ${INFO} | jq -r '.device0 .obs[].humidity') elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then if [[ "${SENSOR}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' elif [[ "${SENSOR}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) temp_h='' if [ "${SENSOR}" == "Powerwall-FAN" ] && [ ${temp_c} -lt 1000 ]; then START_POWERWALL_FAN >/dev/null 2>&1 & fi fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g') temp_c=`echo "scale=1; ${temp_c}/10" | bc` temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g') temp_warn=`echo "scale=1; ${temp_warn}/10" | bc` temp_warn=$(echo "scale=2; ${temp_warn}*1.8 + 32" | bc) temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g') temp_crit=`echo "scale=1; ${temp_crit}/10" | bc` temp_crit=$(echo "scale=2; ${temp_crit}*1.8 + 32" | bc) temp_crit_sys=${temp_crit} temp_h='' vSENSOR=${SENSOR} SENSOR="${SENSORa}-${vSENSOR// /_}" ((t++)) elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi temp_h='' fi [ ! -f ${PW_TMPFOLDER}/.lasttemp/${SENSOR} ] && echo 0 > ${PW_TMPFOLDER}/.lasttemp/${SENSOR} last_temp=$(cat ${PW_TMPFOLDER}/.lasttemp/${SENSOR}) logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR} if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] || [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ]; then SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again" echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - Sensor is reporting data again" >> ${logtemp} fi rm -f ${PW_TMPFOLDER}/${SENSOR}* fi if [ "${SENSORa}" == "iDS-vMS-Host6" ] && [ ! -f ${PW_TMPFOLDER}/.lastfancheck/iDS-vMS-Host6.set ]; then [ -f ${PW_TMPFOLDER}/.lastfancheck/iDS-vMS-Host7.set ] && nfs=$(cat ${PW_TMPFOLDER}/.lastfancheck/iDS-vMS-Host7.set) || nfs=35 SET_ESXI_FANSPEED ${nfs} ${SENSORa} >/dev/null 2>&1 & fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then GAUGE="CPU-USAGE" GAUGESH='%' elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then GAUGE="RPM" GAUGESH="'R" else GAUGE="TEMP" GAUGESH="'F" fi if [ "${SENSOR}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi # echo "HERE: [${SENSOR//-/}] - ${last_temp} - ${temp_c} - ${temp_f} - ${temp_warn} - ${temp_crit} - ${temp_crit_sys}" # [ ! "${temp_c}" ] || [ ! "${temp_f}" ] || [ ! "${temp_warn}" ] || [ ! "${temp_crit}" ] || [ ! "${temp_crit_sys}" ] && echo "VALUE NOT FOUND" if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ]; then echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys ##### ESXI FAN CONTROL ##### elif [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]] && [[ "${vSENSOR}" = *"Exhaust"* ]] && [ "${FanSpeed}" != "" ]; then ### GOLDILOCK ZONE ### if ((( $(bc <<< "${temp_f} >= ${GL_LOW}") )) && (( $(bc <<< "${temp_f} <= ${GL_HIGH}") ))) || [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ]; then if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.gl ] && [ -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set)) -ge 3600 ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN CONTROL" "Goldilock Zone Found at ${FanSpeed}%" echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.gl [ "${SENSORa}" == "iDS-vMS-Host7" ] && [ ! -f ${PW_TMPFOLDER}/iDS-vMS-Host6.down ] && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/iDS-vMS-Host6.gl echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSORa} - Goldilock Zone Found (1HR)" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSORa} - Goldilock Zone Found (1HR)" >> ${PW_LOGFOLDER}/esxi-fanspeeds.log # elif [ -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.gl ] && [ ${FanSpeed} -gt ${ESXI_DEF_FANSPEED} ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.gl)) -ge 10800 ]; then # SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN CONTROL" "DAAAMMMNN GOLDIE!!!! Decreasing by 5 - ${FanSpeed}% --> $(expr ${FanSpeed} - 5)%" # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSORa} - DAAAMMMNN GOLDIE!!!! Decreasing by 5 - ${FanSpeed}% --> $(expr ${FanSpeed} - 5)%" >> ${logtemp} # echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSORa} - DAAAMMMNN GOLDIE!!!! Decreasing by 5 - ${FanSpeed}% --> $(expr ${FanSpeed} - 5)%" >> ${PW_LOGFOLDER}/esxi-fanspeeds.log # SET_ESXI_FANSPEED $(expr $(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set) - 5) ${SENSORa} >/dev/null 2>&1 & elif [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FanSpeed Control" "Defaulting FanSpeeds to ${ESXI_DEF_FANSPEED}%" SET_ESXI_FANSPEED ${ESXI_DEF_FANSPEED} ${SENSORa} >/dev/null 2>&1 & fi ### INCREASING ### elif (( $(bc <<< "${temp_f} >= ${ESXI_DEFAULT_EXHAUST_MIN}") )); then (( $(bc <<< "${temp_f} >= $(expr ${ESXI_DEFAULT_EXHAUST_MIN} + 5)") )) && FAN_INCREASE=10 || FAN_INCREASE=5 [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && NEW_FanSpeed=$(expr ${ESXI_DEF_FANSPEED} + ${FAN_INCREASE}) || NEW_FanSpeed=$(expr $(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set) + ${FAN_INCREASE}) if [ ${NEW_FanSpeed} -le $(expr ${PW_ESXI_MAX_FANSPEED} + 4) ] && [ ${FanSpeed} -ne ${PW_ESXI_MAX_FANSPEED} ]; then [ ${NEW_FanSpeed} -gt ${PW_ESXI_MAX_FANSPEED} ] && NEW_FanSpeed=${PW_ESXI_MAX_FANSPEED} [ ${NEW_FanSpeed} -ge 60 ] && SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Warm" "Increasing fans to ${NEW_FanSpeed}%" SET_ESXI_FANSPEED ${NEW_FanSpeed} ${SENSORa} >/dev/null 2>&1 & fi ### DECREASING ### else (( $(bc <<< "${temp_f} <= $(expr ${ESXI_DEFAULT_EXHAUST_MIN} - 5)") )) && FAN_DECREASE=10 || FAN_DECREASE=5 [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && NEW_FanSpeed=$(expr ${ESXI_DEF_FANSPEED}) || NEW_FanSpeed=$(expr $(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set) - ${FAN_DECREASE}) if [ ${NEW_FanSpeed} -ge $(expr ${ESXI_DEF_FANSPEED} - 4) ] && [ ${FanSpeed} -ne ${ESXI_DEF_FANSPEED} ]; then [ ${NEW_FanSpeed} -lt ${ESXI_DEF_FANSPEED} ] && NEW_FanSpeed=${ESXI_DEF_FANSPEED} [ ${NEW_FanSpeed} -ge 65 ] && SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Cooling" "Decreasing fans to ${NEW_FanSpeed}%" SET_ESXI_FANSPEED ${NEW_FanSpeed} ${SENSORa} >/dev/null 2>&1 & fi fi elif [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]] && [[ "${vSENSOR}" = *"Exhaust"* ]]; then if [ -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FanSpeed Control" "Resetting Fans back to being Host Controlled" rm -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}* ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x01 >/dev/null 2>&1 [ "${SENSORa}" == "iDS-vMS-Host7" ] && ipmitool -I lanplus -H 10.10.2.26 -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x01 >/dev/null 2>&1 elif [ "${FanSpeed}" != "" ]&& [ ${FanSpeed} -ge 90 ] && [ ! -f ${PW_TMPFOLDER}/temp.* ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1 fi fi if [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]] && (( $(bc <<< "${temp_f} < ${temp_warn}") )); then if (( $(bc <<< "${last_temp} >= 1") )); then [ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH} Previous Temp: ${last_temp}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp}${GAUGESH}" >> ${logtemp} # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off elif [ "${last_temp}" == "0" ]; then # SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup # NORMAL TEMP: ${temp_f}${GAUGESH}" echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp} # echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off elif [ "${relog}" == "1" ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Normal ${GAUGE}" >> ${logtemp} fi ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && rm -f ${PW_TMPFOLDER}/temp.* rm -f ${PW_TMPFOLDER}/power.${SENSORa}.* if [ "${last_temp}" != "1" ]; then last_temp=1 echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi elif [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]]; then if ([ ! "${last_temp}" ] || (( $(bc <<< "${last_temp} < 2") ))); then if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi fi temp_diff=`echo "scale=2; ${last_temp}-${temp_f}" | bc` [ $(bc -l <<< "${temp_diff} < 0") -eq 1 ] && temp_diff=`echo "scale=2; ${temp_diff}*-1" | bc` if [ $(bc -l <<< "${temp_f} >= ${temp_warn}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit}") -eq 1 ]; then # ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && touch ${PW_TMPFOLDER}/temp.warn touch ${PW_TMPFOLDER}/temp.warn.${SENSOR} if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} ${GAUGE} WARNING" "WARNING ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - WARNING ${GAUGE} - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp} fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} ${GAUGE} CRITICAL" "CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - CRITICAL - (Difference of ${temp_diff}${GAUGESH})" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} - ${temp_f}${GAUGESH} - CRITICAL ${GAUGE} - (Difference of ${temp_diff}${GAUGESH})" >> ${PW_LOGFILE} if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi # if ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && [ ! -f ${PW_TMPFOLDER}/temp.crit ]; then if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f ${PW_TMPFOLDER}/temp.crit.${SENSOR} ]; then touch ${PW_TMPFOLDER}/temp.crit.${SENSOR} SHUTDOWN_MAIN ${SENSOR} & echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Shutting down MAIN servers" >> ${logtemp} # elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ]; then # SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 # SHUTDOWN_SERVER ${SENSORa} -esxi off & # touch ${PW_TMPFOLDER}/power.${SENSORa}.off fi fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit_sys}") -eq 1 ]; then if (( $(bc <<< "${temp_diff} >= 1") )); then SENDNOTICE "${SENSOR} TEMP CRITICAL" "SYSTEM CRITICAL ${GAUGE}: ${temp_f}${GAUGESH} - (Difference of ${temp_diff}${GAUGESH})" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - CRITICAL ${GAUGE} SYSTEM - (Difference of ${temp_diff}${GAUGESH})" >> ${PW_LOGFILE} if [ "${last_temp}" != "${temp_f}" ]; then last_temp=${temp_f} echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi # if ([ "${SENSOR}" == "ServerRoomTH" ] || [[ "${SENSOR}" = *"ServerRoomLA"* ]]) && [ ! -f ${PW_TMPFOLDER}/temp.critsys ]; then if [ "${SENSOR}" == "ServerRoomTH" ] && [ ! -f ${PW_TMPFOLDER}/temp.critsys.${SENSOR} ]; then touch ${PW_TMPFOLDER}/temp.critsys.${SENSOR} SHUTDOWN_SYS ${SENSOR} TEMP & SHUTDOWN_CRIT ${SENSOR} & echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Sending system shutdown signals, if applicable. Check main Powerwall Logs" >> ${logtemp} fi fi fi fi [ -f ${PW_TMPFOLDER}/.lastinsert/${SENSOR} ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastinsert/${SENSOR})) || lastinsert=100000 if [ $(ls ${PW_TMPFOLDER}/temp.* 2>/dev/null | wc -l) -gt 0 ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.*.low")" != "" ]; then wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_ABNORMAL}*60" | bc) else wait=$(echo "scale=0; ${PW_MYSQL_LOG_INTERVAL_NORMAL}*60" | bc) fi if [ "${PW_SENSOR_ID[${SENSORa}]}" != "" ] && [ ${lastinsert} -ge ${wait} ]; then if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) if [ "${entityid[1]}" == "" ]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${vSENSOR// /_}')" entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='${vSENSOR// /_}' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) fi entityid=${entityid[1]} QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]] && [ "${vSENSOR// /_}" == "System_Board_Inlet" ] && [ "${FanSpeed}" != "" ]; then entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='FanSpeed' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) if [ "${entityid[1]}" == "" ]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO entities (\`sensorid\`, \`name\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','FanSpeed')" entityid=($(${mysql_conn} -e "SELECT id FROM servermonitor.entities WHERE BINARY name='FanSpeed' AND sensorid='${PW_SENSOR_ID[${SENSORa}]}'")) fi entityid=${entityid[1]} ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`entity\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','${entityid}','$(date +'%Y-%m-%d %H:%M:%S')','${FanSpeed}')" fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSORa}" = *"FAN"* ]]; then ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`speed\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" elif [ "${temp_h}" != "" ]; then QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}','${temp_h}')" else QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSORa}]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_f}')" fi ${mysql_conn} -e "${QRY}" # if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "lacrosse" ]; then # ${mysql_conn} -e "USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID["${SENSORa}-Probe"]}','$(date +'%Y-%m-%d %H:%M:%S')','${temp_probe_f}')" # fi echo "${temp_f}" >| ${PW_TMPFOLDER}/.lastinsert/${SENSOR} fi #### LOGGING OF SERVERROOM SENSOR DIFFERENCE if [ "${SENSOR}" == "ServerRoomLA-Probe" ]; then if [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference)) -ge 300 ]; then if [ "${temp_f}" != "" ] && (( $(bc <<< "${temp_f} >= 2") )) && [ -f ${PW_TMPFOLDER}/.serverroomth ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.serverroomth)) -lt 30 ]; then [ ! -f ${PW_TMPFOLDER}/.log.sr.difference ] && touch ${PW_TMPFOLDER}/.log.sr.difference temp_difference=$(bc <<< "scale=2; $(cat ${PW_TMPFOLDER}/.serverroomth)-${temp_f}") # (( $(bc <<<"${temp_difference} < 0") )) && temp_difference=`echo "scale=2; ${temp_difference} * -1" | bc` echo "$(date "+%Y-%m-%d %H:%M:%S")~${temp_difference}" >> ${PW_TMPFOLDER}/.log.sr.difference # echo "[$(date +'%Y-%m-%d %H:%M:%S')] ${temp_f} - $(cat ${PW_TMPFOLDER}/.serverroomth) = ${temp_difference}" rm -f ${PW_TMPFOLDER}/.serverroomth fi fi elif [ "${SENSOR}" == "ServerRoomTH" ]; then echo ${temp_f} >| ${PW_TMPFOLDER}/.serverroomth fi ################### else [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] && touch ${PW_TMPFOLDER}/${SENSOR}.down if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then touch ${PW_TMPFOLDER}/${SENSOR}-error.reading elif [ ! -f ${PW_TMPFOLDER}/${SENSOR}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSOR}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then touch ${PW_TMPFOLDER}/${SENSOR}-error.sent SENDNOTICE "${SENSOR} Sensor ERROR" "ERROR reading sensor data" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} fi fi done fi else [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then touch ${PW_TMPFOLDER}/${SENSORa}-error.reading elif [ ! -f ${PW_TMPFOLDER}/${SENSORa}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then touch ${PW_TMPFOLDER}/${SENSORa}-error.sent SENDNOTICE "${SENSORa} Sensor ERROR" "ERROR reading sensor data" 1 echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp} [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && rm -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set fi fi rm -f ${PW_TMPFOLDER}/ztemprun.${SENSORa}.running touch ${PW_TMPFOLDER}/ztemprun.${SENSORa}.done } SET_ESXI_FANSPEED(){ [ "${PW_ESXI_DEF_FANSPEED}" != "" ] && ESXI_DEF_FANSPEED=${PW_ESXI_DEF_FANSPEED} || ESXI_DEF_FANSPEED=30 SET_FanSpeed=${1} SET_HOST=${2} fan="${FanSpeed}%" if [ ${SET_FanSpeed} -ge ${ESXI_DEF_FANSPEED} ] && [ ${SET_FanSpeed} -le ${PW_ESXI_MAX_FANSPEED} ]; then ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SET_HOST}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SET_HOST}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${SET_FanSpeed}") >/dev/null 2>&1 if [ "${3}" == "manual" ]; then MSG="Manually set Fans to: ${SET_FanSpeed}%" fan=manual elif [ "${SET_FanSpeed}" == "${ESXI_DEF_FANSPEED}" ]; then MSG="Setting Fans back to defaults: ${SET_FanSpeed}%" elif (( $(bc <<< "${SET_FanSpeed} < $(cat ${PW_TMPFOLDER}/.lastfancheck/${SET_HOST}.set)") )); then MSG="Decreasing fans to ${SET_FanSpeed}%" else MSG="Increasing fans to ${SET_FanSpeed}%" fi [ ! -f ${PW_LOGFOLDER}/esxi-fanspeeds.log ] && touch ${PW_LOGFOLDER}/esxi-fanspeeds.log [ "${logtemp}" != "" ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SET_FanSpeed}% - ${SET_HOST} - ${MSG}" >> ${logtemp} [ ${SET_FanSpeed} -ge 70 ] && echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SET_FanSpeed}% - ${SET_HOST} - ${MSG}" >> ${PW_LOGFILE} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${fan} - ${SET_HOST} - ${MSG}" >> ${PW_LOGFOLDER}/esxi-fanspeeds.log echo "${SET_FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SET_HOST}.set [ "${SET_HOST}" == "iDS-vMS-Host7" ] && [ ! -f ${PW_TMPFOLDER}/iDS-vMS-Host6.down ] && SET_ESXI_FANSPEED ${SET_FanSpeed} iDS-vMS-Host6 >/dev/null 2>&1 & [ -f ${PW_TMPFOLDER}/.lastfancheck/${SET_HOST}.gl ] && rm -f ${PW_TMPFOLDER}/.lastfancheck/${SET_HOST}.gl fi } ESXI_FANSPEED_MGMT(){ HOST=${1} FANSPEED=${2} MN=0 if [ "${HOST}" == "" ] || [ "${FANSPEED}" == "" ]; then echo -e "\nMissing arguments\n" exit 1 fi echo -e "${idsCL[Yellow]}Setting new FanSpeeds for the following ESXi Hosts:" DIVIDER . yellow ${PRI_WIDTH} if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]} ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]} elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then ESXI_HOST=${HOST} elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]} elif ([ "${HOST::1}" == "o" ] || [ "${HOST::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]} elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]} elif [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ] || [ "${HOST^^}" == "CONSERVE" ]; then ESXI_HOST=${HOST^^} MN=1 else ESXI_HOST='' fi if [ "${ESXI_HOST}" == "" ]; then echo -e "Invalid host\n" exit 1 fi if [ ${MN} -eq 1 ]; then [ "${ESXI_HOST}" == "ALL" ] && hostname="All Host Servers" || hostname="All ${ESXI_HOST} Host Servers" ESXI_HOSTS=PW_${ESXI_HOST}_HOSTS; ESXI_HOSTS=${!ESXI_HOSTS} else tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} idracip=${PW_HOST_IDRACS[${tmp}]} hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]} fi h=1 for host in ${PW_ESXI_FANSPEEDCONTROLHOSTS[@]}; do tmp=temp if ([ ${MN} -eq 1 ] && [[ "${ESXI_HOSTS}" = *"${PW_HOST_ADDRESSES[${PW_iDRACHOST_SHORTNAMES[${host}]}]}"* ]]) || ([ "${idracip}" != "" ] && [ "${idracip}" == "${host}" ]); then tmp=temp echo -en "${idsCL[LightCyan]}${1}) ${PW_iDRACHOST_SHORTNAMES[${host}]} [${PW_HOST_ADDRESSES[${PW_iDRACHOST_SHORTNAMES[${host}]}]}]" [ -f ${PW_TMPFOLDER}/${PW_iDRACHOST_SHORTNAMES[${host}]}.down ] && echo " - Offline, skipping" || echo fi done DIVIDER true yellow ${PRI_WIDTH} for host in ${PW_ESXI_FANSPEEDCONTROLHOSTS[@]}; do tmp=temp if ([ ${MN} -eq 1 ] && [[ "${ESXI_HOSTS}" = *"${PW_HOST_ADDRESSES[${PW_iDRACHOST_SHORTNAMES[${host}]}]}"* ]]) || ([ "${idracip}" != "" ] && [ "${idracip}" == "${host}" ]); then echo -en "${idsCL[Yellow]}Setting new FanSpeeds for ${PW_iDRACHOST_SHORTNAMES[${host}]} [${PW_HOST_ADDRESSES[${PW_iDRACHOST_SHORTNAMES[${host}]}]}]: " if [ ! -f ${PW_TMPFOLDER}/${PW_iDRACHOST_SHORTNAMES[${host}]}.down ]; then SET_ESXI_FANSPEED ${FANSPEED} ${PW_iDRACHOST_SHORTNAMES[${host}]} manual #>/dev/null 2>&1 echo -e "${idsCL[Green]}FanSpeed set to ${FANSPEED}${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Skipping, host offline${idsCL[Default]}" fi fi done echo } SENDTEMPS(){ st_temps=$(SENDTEMP ${2}) SENDNOTICE "${1}" "${st_temps}" } SENDTEMP(){ echo echo -en "Sensor Temperature(s)" [ "${1}" != "" ] && echo " Now / 1DayAVG / 7DayAVG" || echo DIVIDER . . 40 for SENSORa in ${PW_SENSOR_ORD[@]}; do ([[ "${SENSORa}" = *"-Probe"* ]] || [[ "${SENSORa}" = *"-Usage"* ]]) && cw=25 || cw=30 if [ "${PW_SENSOR_TYPE[${SENSORa}]}" != "esxi" ] && ([ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ] || ([ "${PW_REMOTE_SENSORS[${SENSORa}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSORa}]})" != "false" ])); then SENSORS_CHECK=(${SENSORa}) for SENSOR in "${SENSORS_CHECK[@]}"; do if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then GAUGESH="'%" elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then GAUGESH="'R" else GAUGESH="'F" fi [ "${last_sensor}" != "" ] && [ "${last_sensor}" != "${SENSOR%%-*}" ] && echo if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "mqtt" ]; then mqtt_message=`${mqtt_conn} -t tele/${SENSORa}/SENSOR -C 1` echo ${mqtt_message} | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "lacrosse" ]; then INFO=$(curl -m 5 -s --get -d "deviceid=${PW_LACROSSE_ID[${SENSOR}]}" -d "limit=1" -d "metric=1" -d "timezone=2" https://decent-destiny-704.appspot.com/laxservices/device_info.php | jq) if [[ "${SENSORa}" = *"-Probe" ]]; then temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].probe_temp') 2 yes` else temp_c=`IDS_NUMBER_FORMAT $(echo ${INFO} | jq -r '.device0 .obs[].ambient_temp') 2 yes` fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "system" ]; then if [[ "${SENSORa}" = *"CPU"* ]]; then temp_c=$(vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} elif [[ "${SENSORa}" = *"FAN"* ]]; then temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} vcgencmd measure_temp) if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*} temp_c=${temp_c#*=} fi # elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then # temp_c=${2} # temp_warn=`echo $(bc <<< "scale=2; (${3}*1.8+32)/1") | awk '{print int($1+0.5)}'` # temp_crit=`echo $(bc <<< "scale=2; (${4}*1.8+32)/1") | awk '{print int($1+0.5)}'` elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "cpu" ]; then if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi else temp_c='' fi if [ "${1}" != "" ]; then if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then average1=$(AVERAGETEMP ${SENSORa} 1 ${vSENSOR// /_}) average7=$(AVERAGETEMP ${SENSORa} 7 ${vSENSOR// /_}) # average7=0 else average1=$(AVERAGETEMP ${SENSORa} 1) average7=$(AVERAGETEMP ${SENSORa} 7) # average7=0 fi fi c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then if [ "${SENSOR}" == "ServerRoomTH" ]; then temp_f=`echo "scale=2; ${temp_c}*1.8+32+${PW_ServerRoomTH_Dev}" | bc` else if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "cpu" ]; then temp_f=$(echo "scale=2; $temp_c/1" | bc -l) elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f=${temp_c} else temp_f=$(bc <<< "scale=2; ${temp_c}*1.8+32") fi (( $(bc <<<"${temp_f} < 1") )) && [[ "${temp_f}" = *"."* ]] && temp_f="0${temp_f}" fi if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ] && [[ "${SENSOR}" = *"FAN"* ]]; then temp_f_disp=$(IDS_NUMBER_FORMAT ${temp_f}) if [ "${1}" != "" ]; then average1_disp=$(IDS_NUMBER_FORMAT ${average1}) average7_disp=$(IDS_NUMBER_FORMAT ${average7}) fi else temp_f_disp=${temp_f} if [ "${1}" != "" ]; then average1_disp=${average1} average7_disp=${average7} fi fi [ "${1}" != "" ] && echo -e "${SENSOR}${spc}${temp_f_disp}${GAUGESH} / ${average1_disp}${GAUGESH} / ${average7_disp}${GAUGESH}" || echo -e "${SENSOR}${spc}${temp_f_disp}${GAUGESH}" else echo -e "${SENSOR}${spc}No Data" fi last_sensor=${SENSORa%%-*} done fi done } GET_LACROSSE_INFO(){ INFO=$(/bin/python3 ${PW_INCFOLDER}/getInfo.lacrosse.py "${PW_LACROSSE_ID[${1}]}" | sed -e "s/\[{'//g" | sed -e "s/}\]//g" | sed -e "s/', '/~/g" | sed -e "s/': '/:/g" | sed -e "s/': /:/g") IFS=$'~'; read -rd '' -a INFO <<<"${INFO}"; unset IFS; declare -A DEVICE_INFO for DINFOtmp in "${INFO[@]}"; do key=${DINFOtmp%%:*}; val=${DINFOtmp##*:}; val=${val//[$'\t\r\n ']}; DEVICE_INFO[${key}]=${val}; done echo ${DEVICE_INFO['probe_temp']} } SERVERROOM_TEMP_DIFFERENCE(){ start=$(date +%s) scandefault=288 while [ $# -gt 0 ]; do case "${1}" in -l|-last) LAST=${2};; -h|-help|--help) echo -e "Usage: ${idsCL[Yellow]}[powerwall or pw] tempdiff {opt}${idsCL[Default]} {" width=15 printf "%-${width}s- %s\n" " -l|-last" "(number of last scan times to average from, defaults to ${scandefault})" echo -e "}\n" exit 0;; esac shift done [ "${LAST}" == "" ] && LAST=${scandefault} echo -en "${idsCL[Yellow]}Pulling data and calculating average temperature differences for the last '${LAST}' scans ... " last_tempdiff=$(tail -n 1 ${PW_TMPFOLDER}/.log.sr.difference) last_tempdiff=${last_tempdiff#*~} # if [ $(bc -l <<< "${last_tempdiff} < 1") -eq 1 ]; then [ "${last_tempdiff}" = "0" ] && last_tempdiff="0.0" || last_tempdiff="0${last_tempdiff}"; fi last_tempdiff=`IDS_NUMBER_FORMAT ${last_tempdiff} 2 yes` tempdiffs=$(tail -n ${LAST} ${PW_TMPFOLDER}/.log.sr.difference) largest_tempdiff=0 IFS=$'\n' read -rd '' -a tempdiffs <<<"${tempdiffs}" unset IFS tempdiff_total=0; scansfound=0 tempdiff_large_total=0; scansfound_large=0 for tempdiff in "${tempdiffs[@]}"; do [[ "${tempdiff}" = *"~"* ]] && tempdiff=${tempdiff#*~} if [ $(bc -l <<< "${tempdiff} <= 15") -eq 1 ]; then # (( $(bc <<<"${tempdiff} < 0") )) && tempdiff_tmp1=`echo "scale=2; ${tempdiff} * -1" | bc` || tempdiff_tmp1=${tempdiff} # (( $(bc <<<"${largest_tempdiff} < 0") )) && tempdiff_tmp2=`echo "scale=2; ${largest_tempdiff} * -1" | bc` || tempdiff_tmp2=${largest_tempdiff} # [ $(bc -l <<< "${tempdiff_tmp1} >= ${tempdiff_tmp2}") -eq 1 ] && largest_tempdiff=${tempdiff} # (( $(bc <<<"${tempdiff} < 0") )) && tempdiff_tmp1=`echo "scale=2; ${tempdiff} * -1" | bc` || tempdiff_tmp1=${tempdiff} # [ $(bc -l <<< "${tempdiff_tmp1} >= ${largest_tempdiff}") -eq 1 ] && largest_tempdiff=${tempdiff_tmp1} # [ $(bc -l <<< "${tempdiff} >= ${largest_tempdiff}") -eq 1 ] && largest_tempdiff=${tempdiff} tempdiff_total=$(bc <<< "scale=2; ${tempdiff_total}+${tempdiff}") ((scansfound++)) else tempdiff_large_total=$(bc <<< "scale=2; ${tempdiff_large_total}+${tempdiff}") ((scansfound_large++)) fi done # largest_tempdiff=`IDS_NUMBER_FORMAT ${largest_tempdiff} 2 yes` [ ${scansfound} -eq 0 ] && average=0 || average=$(bc <<< "scale=2; ${tempdiff_total}/${scansfound}") [ ${scansfound_large} -eq 0 ] && average_large=0 || average_large=$(bc <<< "scale=2; ${tempdiff_large_total}/${scansfound_large}") # if [ $(bc -l <<< "${average} < 1") -eq 1 ]; then [ "${average}" = "0" ] && average="0.0" || average="0${average}"; fi average=`IDS_NUMBER_FORMAT ${average} 2 yes` [ -f ${PW_TMPFOLDER}/.log.sr.difference ] && last_scancheck=$(date -d @$(stat -c %Y ${PW_TMPFOLDER}/.log.sr.difference) "+%Y-%m-%d %H:%M:%S") || last_scancheck="" echo -en "\r\033[K" echo -e "${idsCL[LightCyan]}Scan Type Scans Average Temp Difference${idsCL[Default]}" DIVIDER . lightCyan 60 echo -e "${idsCL[Green]}Average for last `IDS_NUMBER_FORMAT ${scansfound}` ${idsCL[LightGreen]}${average}'F${idsCL[Default]}" DIVIDER . lightCyan 60 echo -e "${idsCL[Cyan]} - Large runs `IDS_NUMBER_FORMAT ${scansfound_large}` ${idsCL[LightCyan]}${average_large}'F${idsCL[Default]}" echo # echo -e "${idsCL[LightCyan]}Largest Temp Difference : ${idsCL[LightGreen]}${largest_tempdiff}'F${idsCL[Default]}" # echo echo -e "${idsCL[LightCyan]}Last Temp Difference : ${idsCL[LightGreen]}${last_tempdiff}'F${idsCL[Default]}" echo -e "${idsCL[LightCyan]}Last Check : ${idsCL[LightGreen]}${last_scancheck}${idsCL[Default]}" echo echo -e "${idsCL[LightCyan]}Current Temp Deviation : ${idsCL[LightRed]}${PW_ServerRoomTH_Dev}'F${idsCL[Default]}" if [ ${scansfound} -le 500 ]; then if [ ${scansfound} -gt 15 ]; then echo -en "\n${idsCL[LightCyan]}List out these '${scansfound}' scan times? (y/N): ${idsCL[Default]}" read -n1 choice if [ "${choice^^}" == "Y" ]; then echo -en "\r\033[K" gofor=1 else gofor=0 fi else gofor=1 echo fi if [ ${gofor} -eq 1 ]; then lastscansnum=${scansfound} last_tempdiffs=$(tail -n ${lastscansnum} ${PW_TMPFOLDER}/.log.sr.difference) IFS=$'\n' read -rd '' -a last_tempdiffs <<<"${last_tempdiffs}" unset IFS echo -e "${idsCL[LightCyan]}Last ${lastscansnum} Scan Times\n${idsCL[Green]}------------------------------------------${idsCL[Default]}" for tempdiff in "${last_tempdiffs[@]}"; do if [[ "${tempdiff}" = *"~"* ]]; then dt="${tempdiff%~*}" tempdiff=${tempdiff#*~} # if [ $(bc -l <<< "${tempdiff} < 1") -eq 1 ]; then [ "${tempdiff}" = "0" ] && tempdiff="0.0" || tempdiff="0${tempdiff}"; fi tempdiff=`IDS_NUMBER_FORMAT ${tempdiff} 2` else dt=" " fi echo -e "${idsCL[White]} ${dt} : ${idsCL[Yellow]}${tempdiff}'F${idsCL[Default]}" done fi fi end=`date +%s`; runtime=$((end-start)) echo -e "\nruntime: ${runtime}s\n" }