diff --git a/defaults.inc b/defaults.inc index 2fdbcfb1..c45ed6ea 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='2.3.1-10222023' +VERS='2.3.3-10222023' noheader=' update service dailytemp ' diff --git a/powerwall.sh b/powerwall.sh index 1d602bd5..dfc8f534 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -288,21 +288,47 @@ CHECKTEMP_SERVICE(){ last_temp[${SENSOR}]=$temp_f echo "($datetime) - ${temp_f}'F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp} - - + # iDS-vMS-Offsite Fan Issue Workaround + if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then + vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) + FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') + if [ ${FanSpeed} -ne 80 ]; then + SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30(percent)" 1 + ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x01 0x00" >/dev/null 2>&1 + ssh root@${PW_RACADM_ACCESS} "ipmitool -I lanplus -H ${PW_HOST_IDRACS[${SENSORa}]} -U ${PW_ESXI_USER} -P '${PW_ESXI_PASS}' raw 0x30 0x30 0x02 0xff 0x50" >/dev/null 2>&1 + fi + fi fi elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.crit if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}') - Shutting down servers!!" 1 +Shutting down servers!!" 1 echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=$temp_f SHUTDOWN_MAIN ${SENSOR} & echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} + + # iDS-vMS-Offsite Fan Issue Workaround + if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then + if [ "${TEST}" != "true" ]; then + SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 + ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e true -t 0' & + ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 + ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' + ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e false -t 0' + + while ping -qw 10 -c3 "${PW_HOST_ADDRESSES[${SENSORa}]}">/dev/null 2>&1; do + sleep 1 + done + fi + fi fi elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then touch ${PW_TMPFOLDER}/temp.critsys @@ -319,7 +345,7 @@ CHECKTEMP_SERVICE(){ if [ ${last_temp[${SENSOR}]} -gt 1 ]; then SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F - Previous Temp: ${last_temp[${SENSOR}]}'F" +Previous Temp: ${last_temp[${SENSOR}]}'F" echo -e "($datetime) - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp} echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - Back to NORMAL TEMP" >> ${PW_LOGFILE} [ "$(compgen -G "${PW_FOLDER}/shutdown.*")" != "" ] && rm -f ${PW_FOLDER}/shutdown.*