This commit is contained in:
2023-10-22 21:26:20 -05:00
parent d138271b52
commit 8197d5b7c2
2 changed files with 31 additions and 5 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
VERS='2.3.1-10222023'
VERS='2.3.3-10222023'
noheader=' update service dailytemp '

View File

@@ -288,21 +288,47 @@ CHECKTEMP_SERVICE(){
last_temp[${SENSOR}]=$temp_f
echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1)
FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g')
if [ ${FanSpeed} -ne 80 ]; then
SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30(percent)" 1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00" >/dev/null 2>&1
ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x50" >/dev/null 2>&1
fi
fi
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.crit
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
# iDS-vMS-Offsite Fan Issue Workaround
if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then
if [ "${TEST}" != "true" ]; then
SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e true -t 0' &
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e false -t 0'
while ping -qw 10 -c3 "${PW_HOST_ADDRESSES[${SENSORa}]}">/dev/null 2>&1; do
sleep 1
done
fi
fi
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
touch ${PW_TMPFOLDER}/temp.critsys
@@ -319,7 +345,7 @@ CHECKTEMP_SERVICE(){
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
Previous Temp: ${last_temp[${SENSOR}]}'F"
Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE}
[ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*