diff --git a/defaults.inc b/defaults.inc index 1d6f3808..11a8df4e 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='2.5.165-01272024' +VERS='2.5.166-01272024' noheader=' update service dailytemp confsync -r -report ' diff --git a/inc/temp.inc.sh b/inc/temp.inc.sh index 41b9eba3..4af00476 100644 --- a/inc/temp.inc.sh +++ b/inc/temp.inc.sh @@ -327,7 +327,7 @@ CHECKTEMP_SERVICE(){ DEFAULT_ESXI_FANSPEED=40 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - 'Check Temp' Service Startup" >> ${PW_LOGFILE} SENDTEMPS "TEMP Service Startup" & - [ ! -d ${PW_TMPFOLDER}/.lastfancheck ] && mkdir ${PW_TMPFOLDER}/.lastfancheck + [ ! -d ${PW_TMPFOLDER}/.lastfancheck ] && mkdir ${PW_TMPFOLDER}/.lastfancheck || rm -f ${PW_TMPFOLDER}/.lastfancheck/*.set [ ! -d ${PW_TMPFOLDER}/.lastinsert ] && mkdir ${PW_TMPFOLDER}/.lastinsert [ -d ${PW_TMPFOLDER}/.lasttemp ] && rm -Rf ${PW_TMPFOLDER}/.lasttemp mkdir ${PW_TMPFOLDER}/.lasttemp @@ -400,7 +400,7 @@ CHECKTEMP_SERVICE_RUNSENSOR(){ [ -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off t=1 - ####### Fan Issue Workaround ####### + ####### Grab Fan Speeds ####### if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]); then # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) @@ -544,8 +544,52 @@ CHECKTEMP_SERVICE_RUNSENSOR(){ echo ${PW_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys if [[ "${vSENSOR}" = *"Exhaust"* ]]; then - if (( $(bc <<< "${temp_f} >= 100") )); then - temp=temp + if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then + if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set)) -ge 600 ]; then + if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && (( $(bc <<< "${temp_f} < 100") )); then + if [ ${FanSpeed} -ne ${DEFAULT_ESXI_FANSPEED} ]; then + if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then + SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FanSpeed Control" "Settings fans back to normal ${DEFAULT_ESXI_FANSPEED}%" + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${DEFAULT_ESXI_FANSPEED}") >/dev/null 2>&1 + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${PW_LOGFILE} + echo "${DEFAULT_ESXI_FANSPEED}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set + elif [ ${FanSpeed} -ge 90 ]; then + SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1 + fi + fi + ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} + echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set + + elif (( $(bc <<< "${temp_f} >= 100") )); then + [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && NEW_FanSpeed=$(expr ${DEFAULT_ESXI_FANSPEED} + 5) || NEW_FanSpeed=$(expr ${FanSpeed} + 5) + if [ ${NEW_FanSpeed} -le 104 ]; then + [ ${NEW_FanSpeed} -gt 100 ] && NEW_FanSpeed=100 + SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Warm" "Increasing fans to ${NEW_FanSpeed}%" + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${NEW_FanSpeed}") >/dev/null 2>&1 + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Increasing fans to ${NEW_FanSpeed}%" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Increasing fans to ${NEW_FanSpeed}%" >> ${PW_LOGFILE} + echo "${NEW_FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set + fi + + else + NEW_FanSpeed=$(expr ${FanSpeed} - 5) + if [ ${NEW_FanSpeed} -ge $(expr ${DEFAULT_ESXI_FANSPEED} - 4) ]; then + [ ${NEW_FanSpeed} -lt ${DEFAULT_ESXI_FANSPEED} ] && NEW_FanSpeed=${DEFAULT_ESXI_FANSPEED} + SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Exhaust Cooling" "Decreasing fans to ${NEW_FanSpeed}%" + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 + ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${NEW_FanSpeed}") >/dev/null 2>&1 + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Decreasing fans to ${NEW_FanSpeed}%" >> ${logtemp} + echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Decreasing fans to ${NEW_FanSpeed}%" >> ${PW_LOGFILE} + echo "${NEW_FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set + fi + + fi + + [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set ] && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa}.set + fi fi fi @@ -577,25 +621,6 @@ Previous Temp: ${last_temp}${GAUGESH}" echo ${last_temp} >| ${PW_TMPFOLDER}/.lasttemp/${SENSOR} fi - ####### Fan Issue Workaround ####### - if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]; then - if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]; then - if [ ${FanSpeed} -ne ${DEFAULT_ESXI_FANSPEED} ]; then - - if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then - SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Settings fans back to normal ${DEFAULT_ESXI_FANSPEED}%" - ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 - ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x$(printf "%X" "${DEFAULT_ESXI_FANSPEED}") >/dev/null 2>&1 - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${logtemp} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${FanSpeed}% - ${SENSOR} - Resetting FanSpeed back to ${DEFAULT_ESXI_FANSPEED}%" >> ${PW_LOGFILE} - elif [ ${FanSpeed} -ge 90 ]; then - SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} FAN ISSUE?!" "Possible fan issue, currently running at ${FanSpeed}%" 1 - fi - fi - ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} - fi - fi - elif [[ "${SENSORa}" != *"-Usage"* ]] && [[ "${SENSORa}" != *"FAN"* ]]; then if ([ ! "${last_temp}" ] || (( $(bc <<< "${last_temp} < 2") ))); then @@ -621,24 +646,6 @@ Previous Temp: ${last_temp}${GAUGESH}" fi - ####### Fan Issue Workaround ####### - if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ "${FanSpeed}" != "" ]]; then - if [ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]; then - # if [ ${FanSpeed} -lt 26 ] || [ ${FanSpeed} -gt 34 ]; then - if [ ${FanSpeed} -ne 80 ]; then - if [[ "${PW_ESXI_FANSPEEDCONTROL_HOSTS}" = *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then - SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Setting fan speeds to 80%" 1 - ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x01 0x00 >/dev/null 2>&1 - ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" raw 0x30 0x30 0x02 0xff 0x50 >/dev/null 2>&1 - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${logtemp} - echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Server Warm, setting FanSpeed to 80%" >> ${PW_LOGFILE} - else - SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} SERVER WARM" "Fans not set at 80%" 1 - fi - fi - ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ "$(cat ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})" != "${FanSpeed}" ]) && echo "${FanSpeed}" >| ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} - fi - fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}") -eq 1 ] && [ $(bc -l <<< "${temp_f} < ${temp_crit_sys}") -eq 1 ]; then if (( $(bc <<< "${temp_diff} >= 1") )); then