From 70b90eaee55965d66816607897f9d4ca69aa18f0 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Sat, 4 Nov 2023 13:32:32 -0500 Subject: [PATCH] update --- defaults.inc | 2 +- powerwall.sh | 96 ++++++++++++++++++++++------------------------------ 2 files changed, 42 insertions(+), 56 deletions(-) diff --git a/defaults.inc b/defaults.inc index a0624e17..e6467aae 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='2.3.147-11032023' +VERS='2.3.150-11042023' noheader=' update service dailytemp ' diff --git a/powerwall.sh b/powerwall.sh index 8982a506..e0690609 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -218,17 +218,16 @@ CHECKTEMP_SERVICE(){ SENDNOTICE "${SENSORa} Online" "${SENSORa} Sensor is back online" echo "(${datetime}) - ${SENSORa} - Sensor is back online" >> ${logtemp} rm -f ${PW_TMPFOLDER}/${SENSORa}.down - fi if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then vHOST_TEMPNAMES=$(snmpwalk -v 1 -c public -t 2 ${vHOSTiDRACIP} .1.3.6.1.4.1.674.10892.5.4.700.20.1.8 -O vq | sed -e 's/"//g' | sed -e 's/ Temp//g') unset IFS; IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${vHOST_TEMPNAMES}"; unset IFS - - # for i in ${SENSORS_CHECK[@]}; do echo $i; done - + [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.${SENSORa}.off)) -ge 300 ] && rm-f ${PW_TMPFOLDER}/power.${SENSORa}.off + t=1 + # iDS-vMS-Offsite Fan Issue Workaround if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then - vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 10.2.1.21 .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq) + vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.1 -O vq) FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=`echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'` if [ ${FanSpeed} -gt 92 ]; then SENDNOTICE "${PW_HOST_IDRACS[${SENSORa}]} Fan Issue" "Workaround commands sent to set the Fans at 30(percent)" 1 @@ -260,21 +259,18 @@ CHECKTEMP_SERVICE(){ temp_h='' elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then - temp_c=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $2}' | sed -e 's/C//g'); temp_c=${temp_c/ /} - temp_warn=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $7}' | sed -e 's/C//g'); - temp_crit=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); - temp_crit_sys=$(echo ${SENSOR} | cut -d'p' -f2 | awk '/ / {print $4}' | sed -e 's/C//g'); - - temp_warn=`echo $(bc <<< "scale=2; (${temp_warn/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'` - temp_crit=`echo $(bc <<< "scale=2; (${temp_crit/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'` - temp_crit_sys=`echo $(bc <<< "scale=2; (${temp_crit_sys/ /}*1.8+32)/1") | awk '{print int($1+0.5)}'` - + temp_c=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.6.1.${t} -O vq | sed -e 's/"//g') + temp_c=`echo "scale=1; ${temp_c}/10" | bc` + temp_warn=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.11.1.${t} -O vq | sed -e 's/"//g') + temp_warn=`echo "scale=1; ${temp_warn}/10" | bc` + temp_crit=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.20.1.10.1.${t} -O vq | sed -e 's/"//g') + temp_crit=`echo "scale=1; ${temp_crit}/10" | bc` + temp_crit_sys=${temp_crit} temp_h='' vSENSOR=${SENSOR} SENSOR="${SENSORa}-${vSENSOR// /_}" - + ((t++)) fi - datetime=`date +'%Y-%m-%d %H:%M:%S'` logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR} @@ -326,26 +322,16 @@ Shutting down servers!!" 1 echo "(${datetime}) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp} echo -e "(${datetime}) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${PW_LOGFILE} last_temp[${SENSOR}]=${temp_f} - SHUTDOWN_MAIN ${SENSOR} & - echo -e "(${datetime}) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} - # iDS-vMS-Offsite Fan Issue Workaround - if [ "${PW_HOST_IDRACS[${SENSORa}]}" == "10.2.1.21" ]; then - if [ "${TEST}" != "true" ]; then - SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 + if [[ "${PW_OFFSITEHOSTS}" != *"${PW_HOST_IDRACS[${SENSORa}]}"* ]]; then + SHUTDOWN_MAIN ${SENSOR} & + echo -e "(${datetime}) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} + + elif [ ! -f ${PW_TMPFOLDER}/power.${SENSORa}.off ] + SENDNOTICE "${SENSORa} TEMP CRITICAL" "Host system and VMs shutting down!!" 1 + SHUTDOWN_SERVER ${SENSORa} & + touch ${PW_TMPFOLDER}/power.${SENSORa}.off - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 - ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 - ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e true -t 0' & - ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 - ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_HOST_ADDRESSES[${SENSORa}]} >/dev/null 2>&1 - ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' - ssh root@${PW_HOST_ADDRESSES[${SENSORa}]} 'esxcli system maintenanceMode set -e false -t 0' - - while ping -qw 10 -c3 "${PW_HOST_ADDRESSES[${SENSORa}]}">/dev/null 2>&1; do - sleep 1 - done - fi fi fi elif [ $(bc -l <<< "${temp_f} >= ${temp_crit}_sys") -eq 1 ]; then @@ -380,6 +366,7 @@ Previous Temp: ${last_temp[${SENSOR}]}'F" fi rm -f ${PW_TMPFOLDER}/temp.* + rm -f ${PW_TMPFOLDER}/power.${SENSORa}.off last_temp[${SENSOR}]=1 # iDS-vMS-Offsite Fan Issue Workaround @@ -428,6 +415,9 @@ Previous Temp: ${last_temp[${SENSOR}]}'F" fi done + + + fi elif [ "${PW_HOST_ADDRESSES[${SENSORa}]}" == "" ]; then #sensor down @@ -487,13 +477,13 @@ DAILYTEMP(){ temp_c='' fi - # if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then - # average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_}) - # average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_}) - # else - # average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1) - # average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7) - # fi + if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then + average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1 ${vSENSOR// /_}) + average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7 ${vSENSOR// /_}) + else + average1=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 1) + average7=$(AVERAGETEMP ${PW_SENSOR_ID[${SENSORa}]} 7) + fi c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done if [ "${temp_c}" != "null" ] && [ "${temp_c}" != "" ]; then @@ -758,26 +748,13 @@ VOLTAGE: ${OFFSITE_VOLTIN}'V" 1 fi if [ ${OFFSITE_BATT} -le 70 ] && [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then - for host in "${PW_OFFSITEHOSTS[@]}"; do - SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "(${datetime}) Shutting down '${PW_ESXI_HOST_NAMES[${host}]}'" 1 - SHUTDOWN_SERVER ${host} & - touch ${PW_TMPFOLDER}/power.offsite.off - done + SHUTDOWN_OFFSITE & fi fi fi - if [ "${archivedone}" != "complete" ] && [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0000 ] && [ $(date +%H)$(date +%M) -le 0005 ]; then - LOGROTATE & - archivedone=complete - - # elif [ $(date +%d) -eq 1 ] && [ $(date +%H)$(date +%M) -ge 0100 ] && [ "${archivedone}" = "complete" ]; then - # archivedone=false - - fi - sleep 10s done # & } @@ -982,6 +959,15 @@ SHUTDOWN_CRIT(){ fi } +SHUTDOWN_OFFSITE(){ + if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then + for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do + SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "(${datetime}) Shutting down '${PW_ESXI_HOST_NAMES[${offsite_host}]}'" 1 + SHUTDOWN_SERVER ${offsite_host} & + done + touch ${PW_FOLDER}/shutdown.offsite + fi +} SHUTDOWN_SERVERS(){ # TEST=true