diff --git a/defaults.inc b/defaults.inc index 885c4de0..1c052077 100755 --- a/defaults.inc +++ b/defaults.inc @@ -29,6 +29,11 @@ PW_LOGPOWER=${PW_LOGFOLDER}/log-power [ ! -f ${PW_LOGPOWER} ] && touch ${PW_LOGPOWER} +declare -A PW_POWERWALL_SERVICES +PW_POWERWALL_SERVICES['temp']='monitor-temp' +PW_POWERWALL_SERVICES['power']='monitor-power' +# PW_POWERWALL_SERVICES['powerlogger']='monitor-powerlogger' + declare -A PW_TEMP_THRESHOLDS # PW_TEMP_THRESHOLDS['ServerRoomTH']="72,74,76" PW_TEMP_THRESHOLDS['ServerRoomTH']="83,88,95" @@ -39,6 +44,11 @@ declare -A PW_SENSOR_TYPE PW_SENSOR_TYPE['ServerRoomTH']='mqtt' PW_SENSOR_TYPE['RaspberryPI-CPU']='system' PW_SENSOR_TYPE['OctoPI-CPU']='remotesystem' +PW_SENSOR_TYPE['esxi0']='esxi' +PW_SENSOR_TYPE['esxi1']='esxi' +PW_SENSOR_TYPE['esxi2']='esxi' +PW_SENSOR_TYPE['esxi7']='esxi' +PW_SENSOR_TYPE['esxi8']='esxi' declare -A PW_REMOTE_SENSORS PW_REMOTE_SENSORS['ServerRoomTH']='10.10.20.10' @@ -48,18 +58,25 @@ declare -A PW_SENSOR_ID PW_SENSOR_ID['ServerRoomTH']='2' PW_SENSOR_ID['RaspberryPI-CPU']='6' PW_SENSOR_ID['OctoPI-CPU']='7' +# PW_SENSOR_ID['esxi0']='8' +# PW_SENSOR_ID['esxi1']='9' +# PW_SENSOR_ID['esxi2']='10' +# PW_SENSOR_ID['esxi7']='11' +# PW_SENSOR_ID['esxi8']='12' -declare -A PW_POWERWALL_SERVICES -PW_POWERWALL_SERVICES['temp']='monitor-temp' -PW_POWERWALL_SERVICES['power']='monitor-power' -# PW_POWERWALL_SERVICES['powerlogger']='monitor-powerlogger' +declare -A PW_ESXI_MON_HOSTS +PW_ESXI_MON_HOSTS['esxi0']='10.10.2.10' +PW_ESXI_MON_HOSTS['esxi1']='10.10.2.11' +PW_ESXI_MON_HOSTS['esxi2']='10.10.2.12' +PW_ESXI_MON_HOSTS['esxi7']='10.10.2.17' +PW_ESXI_MON_HOSTS['esxi8']='10.2.1.18' declare -A PW_ESXI_HOST_NAMES -PW_ESXI_HOST_NAMES['10.10.2.10']='ids-vms-scsi.scity.us' -PW_ESXI_HOST_NAMES['10.10.2.11']='ids-vms1.scity.us' -PW_ESXI_HOST_NAMES['10.10.2.12']='ids-vms2.scity.us' -PW_ESXI_HOST_NAMES['10.10.2.17']='ids-vms7.scity.us' -PW_ESXI_HOST_NAMES['10.2.1.18']='ids-vms-offsite.scity.us' +PW_ESXI_HOST_NAMES['10.10.2.10']='iDS-vMS-scsi.scity.us' +PW_ESXI_HOST_NAMES['10.10.2.11']='iDS-vMS1.scity.us' +PW_ESXI_HOST_NAMES['10.10.2.12']='iDS-vMS2.scity.us' +PW_ESXI_HOST_NAMES['10.10.2.17']='iDS-vMS7.scity.us' +PW_ESXI_HOST_NAMES['10.2.1.18']='iDS-vMS-Offsite.scity.us' PW_VCENTERHOST=10.10.1.30 PW_MAINHOSTS=10.10.2.10,10.10.2.11,10.10.2.12 @@ -68,6 +85,8 @@ PW_OFFSITEHOSTS=10.2.1.18 PW_MAINSITE_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS}" PW_ALL_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS},${PW_OFFSITEHOSTS}" +PW_RACADM_ACCESS=10.10.0.241 + IFS=, PW_MAINSITE_HOSTS=(${PW_MAINSITE_HOSTS}) PW_MAINHOSTS=(${PW_MAINHOSTS}) diff --git a/powerwall.sh b/powerwall.sh index 1f71dcfa..5d77d85d 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -45,6 +45,7 @@ CHECKTEMP(){ fi [ "${action}" != "" ] && echo } + CHECKTEMPSENSOR(){ echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn echo ${PW_TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit @@ -64,7 +65,36 @@ CHECKTEMPSENSOR(){ temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} fi + elif [ "${PW_SENSOR_TYPE[${1}]}" == "esxi" ]; then + for e in ${!PW_ESXI_MON_HOSTS[@]}; do + vHOSTIP=${PW_ESXI_MON_HOSTS[${e}]} + vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTIP}]} + if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then + vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1) + IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS + + for sensordata in "${vHOSTDATA[@]}"; do + sensor=$(echo ${sensordata} | cut -d'T' -f1 | xargs) + temp_c=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); current=${current/ /} + temp_warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); warn=${warn/ /} + temp_crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); crit=${crit/ /} + + + + done + + + else #host down + if [ ! -f ${PW_TMPFOLDER}/${e}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${e}.down)) -ge 1800 ]; then + SENDNOTICE "${vHOSTIP} ERROR" "${vHOSTNAME} Host is down" 1 + echo "($datetime) - ${vHOSTIP} - ${vHOSTNAME} Host is down" >> ${logtemp} + touch ${PW_TMPFOLDER}/${e}.down + fi + + fi + done + fi if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then @@ -120,145 +150,180 @@ CHECKTEMP_SERVICE(){ errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp})) [ $errtime -ge 3600 ] && relog=1 || relog=0 - if [ "${PW_REMOTE_SENSORS[${SENSOR}]}" == "" ] || ([ "${PW_REMOTE_SENSORS[${SENSOR}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSOR}]})" != "false" ]); then - if [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then - SENDNOTICE "${SENSOR} Online" "${SENSOR} Sensor is back online" - echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is back onlione" >> ${logtemp} - rm -f ${PW_TMPFOLDER}/${SENSOR}.down + if [ "${PW_SENSOR_TYPE[${SENSOR}]}" != "esxi" ] + if [ "${PW_REMOTE_SENSORS[${SENSOR}]}" == "" ] || ([ "${PW_REMOTE_SENSORS[${SENSOR}]}" != "" ] && [ "$(CHECK_HOST ${PW_REMOTE_SENSORS[${SENSOR}]})" != "false" ]); then + if [ -f ${PW_TMPFOLDER}/${SENSOR}.down ]; then + SENDNOTICE "${SENSOR} Online" "${SENSOR} Sensor is back online" + echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is back onlione" >> ${logtemp} + rm -f ${PW_TMPFOLDER}/${SENSOR}.down - fi + fi - if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then - mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` - echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h - echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c + if [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then + mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1` + echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h + echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c - elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ]; then - temp_c=$(vcgencmd measure_temp) - temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} - temp_h='' + elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "system" ]; then + temp_c=$(vcgencmd measure_temp) + temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} + temp_h='' - elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then - temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) - temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} - temp_h='' - - fi - datetime=`date +'%Y-%m-%d %H:%M:%S'` - - if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then - if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then - SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again" - echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp} - rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading + elif [ "${PW_SENSOR_TYPE[${SENSOR}]}" == "remotesystem" ]; then + temp_c=$(ssh root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) + temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} + temp_h='' - fi - temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` + fi + datetime=`date +'%Y-%m-%d %H:%M:%S'` + + if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then + if [ -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ]; then + SENDNOTICE "${SENSOR} Data" "${SENSOR} is reporting data again" + echo "($datetime) - ${SENSOR} - ${SENSOR} is reporting data again" >> ${logtemp} + rm -f ${PW_TMPFOLDER}/${SENSOR}-error.reading + + fi + temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc` - echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn - echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit - echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys + echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn + echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit + echo ${PW_TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys - if [ "${last_temp[${SENSOR}]}" != "" ]; then - temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}` - [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) - else - last_temp[${SENSOR}]=0 - temp_diff=0 - fi - - if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then - touch ${PW_TMPFOLDER}/temp.warn - if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 - echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} - last_temp[${SENSOR}]=$temp_f - echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp} - fi - - elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then - touch ${PW_TMPFOLDER}/temp.crit - if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') - Shutting down servers!!" 1 - echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} - echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} - echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} - last_temp[${SENSOR}]=$temp_f - SHUTDOWN_MAIN ${SENSOR} & - echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} - - fi - elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then - touch ${PW_TMPFOLDER}/temp.critsys - if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then - SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') - Shutting down system servers!!" 1 - echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} - echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} - echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} - last_temp[${SENSOR}]=$temp_f - SHUTDOWN_SYS ${SENSOR} TEMP & - SHUTDOWN_CRIT ${SENSOR} & - echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} - fi - else - [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${PW_TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" & - - if [ ${last_temp[${SENSOR}]} -gt 1 ]; then - SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F - Previous Temp: ${last_temp[${SENSOR}]}'F" - echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} - echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} - echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE} - [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* - - elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then - SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup - NORMAL TEMP: ${temp_f}'F" - echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} - echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} - [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* - - elif [ $relog -eq 1 ]; then - echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp} - - fi - rm -f ${PW_TMPFOLDER}/temp.* - last_temp[${SENSOR}]=1 - fi - - [ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000 - if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then - wait=60 - else - wait=300 - fi - if [ ${lastinsert} -ge ${wait} ]; then - if [ "${temp_h}" != "" ]; then - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" + if [ "${last_temp[${SENSOR}]}" != "" ]; then + temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}` + [ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1)) else - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" + last_temp[${SENSOR}]=0 + temp_diff=0 fi - ${mysql_conn} -e "${QRY}" - touch ${PW_TMPFOLDER}/${SENSOR}.insert - fi - else - if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then - SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1 - echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} - touch ${PW_TMPFOLDER}/${SENSOR}-error.reading - fi + + if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then + touch ${PW_TMPFOLDER}/temp.warn + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1 + echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} + last_temp[${SENSOR}]=$temp_f + echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp} + fi + + elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then + touch ${PW_TMPFOLDER}/temp.crit + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') + Shutting down servers!!" 1 + echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} + echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} + echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} + last_temp[${SENSOR}]=$temp_f + SHUTDOWN_MAIN ${SENSOR} & + echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} + fi + elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then + touch ${PW_TMPFOLDER}/temp.critsys + if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then + SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') + Shutting down system servers!!" 1 + echo "($datetime) - ${temp_f}'F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp} + echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} + echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} + last_temp[${SENSOR}]=$temp_f + SHUTDOWN_SYS ${SENSOR} TEMP & + SHUTDOWN_CRIT ${SENSOR} & + echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} + fi + else + [ ! -f ${PW_TMPFOLDER}/temp.* ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.${SENSOR}.sentdaily)) -gt 600 ] && [ $(date +%H)$(date +%M) -ge 0700 ] && [ $(date +%H)$(date +%M) -lt 0705 ] && touch ${PW_TMPFOLDER}/.${SENSOR}.sentdaily && SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" & + + if [ ${last_temp[${SENSOR}]} -gt 1 ]; then + SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F + Previous Temp: ${last_temp[${SENSOR}]}'F" + echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} + echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} + echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE} + [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* + + elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then + SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup + NORMAL TEMP: ${temp_f}'F" + echo -e "($datetime) - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp} + echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Service Startup" >> ${PW_LOGFILE} + [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.* + + elif [ $relog -eq 1 ]; then + echo "($datetime) - ${temp_f}'F - Normal Temp" >> ${logtemp} + + fi + rm -f ${PW_TMPFOLDER}/temp.* + last_temp[${SENSOR}]=1 + fi + + [ -f ${PW_TMPFOLDER}/${SENSOR}.insert ] && lastinsert=$(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.insert)) || lastinsert=100000 + if [ "$(compgen -G "${PW_TMPFOLDER}/temp.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${PW_TMPFOLDER}/power.dc.*")" != "" ]; then + wait=60 + else + wait=300 + fi + if [ ${lastinsert} -ge ${wait} ]; then + if [ "${temp_h}" != "" ]; then + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" + else + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${PW_SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" + fi + ${mysql_conn} -e "${QRY}" + touch ${PW_TMPFOLDER}/${SENSOR}.insert + fi + else + if [ ! -f ${PW_TMPFOLDER}/${SENSOR}-error.reading ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}-error.reading)) -ge 1800 ]; then + SENDNOTICE "${SENSOR} Sensor ERROR" "($datetime) ERROR reading sensor data" 1 + echo "($datetime) - ${SENSOR} Sensor - ERROR reading sensor data" >> ${logtemp} + touch ${PW_TMPFOLDER}/${SENSOR}-error.reading + fi + + fi + + else #sensor down + if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then + SENDNOTICE "${SENSOR} ERROR" "${SENSOR} Sensor is down" 1 + echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is down" >> ${logtemp} + touch ${PW_TMPFOLDER}/${SENSOR}.down + + fi + fi - else - if [ ! -f ${PW_TMPFOLDER}/${SENSOR}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${SENSOR}.down)) -ge 1800 ]; then - SENDNOTICE "${SENSOR} ERROR" "${SENSOR} Sensor is down" 1 - echo "($datetime) - ${SENSOR} - ${SENSOR} Sensor is down" >> ${logtemp} - touch ${PW_TMPFOLDER}/${SENSOR}.down + ############################# + else #esxi hosts + for e in ${!PW_ESXI_MON_HOSTS[@]}; do + vHOSTIP=${PW_ESXI_MON_HOSTS[${e}]} + vHOSTNAME=${PW_ESXI_HOST_NAMES[${vHOSTIP}]} + if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then + vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1) + IFS=$'\n' read -rd '' -a vHOSTDATA <<<"${vHOSTDATA}"; unset IFS + + for sensordata in "${vHOSTDATA[@]}"; do + sensor=$(echo ${sensordata} | cut -d'T' -f1 | xargs) + temp=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); current=${current/ /} + warn=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); warn=${warn/ /} + crit=$(echo ${sensordata} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); crit=${crit/ /} + + + + done + + + else #host down + if [ ! -f ${PW_TMPFOLDER}/${e}.down ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/${e}.down)) -ge 1800 ]; then + SENDNOTICE "${vHOSTIP} ERROR" "${vHOSTNAME} Host is down" 1 + echo "($datetime) - ${vHOSTIP} - ${vHOSTNAME} Host is down" >> ${logtemp} + touch ${PW_TMPFOLDER}/${e}.down + + fi + + fi + done - fi fi done @@ -268,6 +333,8 @@ CHECKTEMP_SERVICE(){ done # & } + + DAILYTEMP(){ # start=`date +%s` cw=26;