This commit is contained in:
2023-12-26 19:18:15 -06:00
parent b855f76d66
commit f4320f6cce
2 changed files with 315 additions and 293 deletions

View File

@@ -308,8 +308,8 @@ CHECKTEMP_SERVICE(){
# SENDDAILY & # SENDDAILY &
# fi # fi
# fi # fi
SENSORS_CHECKED=()
for SENSORa in ${PW_SENSOR_ORD[@]}; do for SENSORa in ${PW_SENSOR_ORD[@]}; do
echo -en "${SENSORa}"; start_sub=`date +%s`
logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa} logtemp=${PW_LOGFOLDER}/log-temp-${SENSORa}
[ ! -f ${logtemp} ] && touch ${logtemp} [ ! -f ${logtemp} ] && touch ${logtemp}
[ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0 [ $(expr `date +%s` - $(stat -c %Y ${logtemp})) -ge $(echo "scale=0; ${PW_LOG_INTERVAL_NORMAL}*60" | bc) ] && relog=1 || relog=0
@@ -321,6 +321,44 @@ CHECKTEMP_SERVICE(){
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp} echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} - Sensor is back online" >> ${logtemp}
rm -f ${PW_TMPFOLDER}/${SENSORa}.down rm -f ${PW_TMPFOLDER}/${SENSORa}.down
fi fi
CHECKTEMP_SERVICE_RUNSENSOR ${SENSORa} &
SENSORS_CHECKED+=(${SENSORa})
else
[ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then
touch ${PW_TMPFOLDER}/${SENSORa}-error.reading
elif [ ! -f ${PW_TMPFOLDER}/${SENSORa}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
touch ${PW_TMPFOLDER}/${SENSORa}-error.sent
SENDNOTICE "${SENSORa} Sensor ERROR" "ERROR reading sensor data" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp}
fi
fi
done
for SENSORa in "${SENSORS_CHECKED[@]}"; do
[ ! -f ${dli//.down/.disable} ] && SENSORS_CHECKED+=(${SENSORa})
done
s_count=0; completed=false
until [ "${completed}" == "true" ]; do
for sensor in ${PW_TMPFOLDER}/zservicerun.*.done; do
SENSORa=$(grep -oP '(?<=zservicerun.).*?(?=.done)' <<< "${nodestatus}")
if [ "${SENSORa}" != "*" ]; then
rm -f ${sensor}
((s_count++))
# echo "${NTS} == ${s_count} == ${nodestatus}"
fi
done
[ ${s_count} -eq ${#SENSORS_CHECKED[@]} ] && completed=true
done
end=`date +%s`; runtime=$((end-start)); echo "Total runtime: ${runtime}s"
# sleep 10s
done # &
${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
}
CHECKTEMP_SERVICE_RUNSENSOR(){
SENSORa=${1}
if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g')
IFS=$'\n' IFS=$'\n'
@@ -456,14 +494,14 @@ CHECKTEMP_SERVICE(){
if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then if (( $(bc <<< "${temp_f} < ${temp_warn}") )); then
if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then if (( $(bc <<< "${last_temp[${SENSOR//-/}]} > 1") )); then
[ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH} [ -f ${PW_TMPFOLDER}/${SENSORa}.down ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}.down)) -ge 1800 ] && SENDNOTICE "${SENSOR} BACK TO NORMAL" "NORMAL TEMP: ${temp_f}${GAUGESH}
Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}" Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Back to NORMAL ${GAUGE} - Previous ${GAUGE}: ${last_temp[${SENSOR//-/}]}${GAUGESH}" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Back to NORMAL ${GAUGE}" >> ${PW_LOGFILE}
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then elif [ "${last_temp[${SENSOR//-/}]}" == "0" ]; then
# SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup # SENDNOTICE "${SENSOR} TEMP NORMAL" "Service Startup
# NORMAL TEMP: ${temp_f}${GAUGESH}" # NORMAL TEMP: ${temp_f}${GAUGESH}"
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - Service Startup - NORMAL ${GAUGE}" >> ${logtemp}
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}${GAUGESH} - ${SENSOR} - Service Startup" >> ${PW_LOGFILE}
# [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off # [ "$(compgen -G "${PW_TMPFOLDER}/power.*.off")" != "" ] && rm -f ${PW_TMPFOLDER}/power.*.off
@@ -609,23 +647,7 @@ Previous Temp: ${last_temp[${SENSOR//-/}]}${GAUGESH}"
fi fi
done done
fi fi
# elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down touch ${PW_TMPFOLDER}/zservicerun.${SENSORa}.done
else
[ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && touch ${PW_TMPFOLDER}/${SENSORa}.down
if [ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.reading ]; then
touch ${PW_TMPFOLDER}/${SENSORa}-error.reading
elif [ ! -f ${PW_TMPFOLDER}/${SENSORa}.disable ] && (([ ! -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.reading)) -ge 60 ]) || ([ -f ${PW_TMPFOLDER}/${SENSORa}-error.sent ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSORa}-error.sent)) -ge $(echo "scale=0; ${PW_RENOTIFY_TIMEOUT}*60" | bc) ])); then
touch ${PW_TMPFOLDER}/${SENSORa}-error.sent
SENDNOTICE "${SENSORa} Sensor ERROR" "ERROR reading sensor data" 1
echo "($(date +'%Y-%m-%d %H:%M:%S')) - ${SENSORa} Sensor - ERROR reading sensor data" >> ${logtemp}
fi
fi
end_sub=`date +%s`; runtime_sub=$((end_sub-start_sub)); echo " - done: ${runtime_sub}s"
done
end=`date +%s`; runtime=$((end-start)); echo "Total runtime: ${runtime}s"
# sleep 10s
done # &
} }
DAILYTEMP(){ DAILYTEMP(){

View File

@@ -79,7 +79,7 @@ CHECK_SERVICES(){
} }
SHUTDOWN_MAIN(){ SHUTDOWN_MAIN(){
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
touch ${PW_TMPFOLDER}/shutdown.MAIN.started touch ${PW_TMPFOLDER}/shutdown.MAIN.started
SHUTDOWN_SERVER main -esxi off SHUTDOWN_SERVER main -esxi off
[ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!' [ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!'
@@ -98,7 +98,7 @@ ${reason}" 1
echo echo
} }
SHUTDOWN_SYS(){ SHUTDOWN_SYS(){
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "ServerRoomLA" ] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then
if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ]; then if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ]; then
touch ${PW_TMPFOLDER}/shutdown.MAINSITE.started touch ${PW_TMPFOLDER}/shutdown.MAINSITE.started
SHUTDOWN_SERVER mainsite -esxi off SHUTDOWN_SERVER mainsite -esxi off