diff --git a/defaults.inc b/defaults.inc index e68beca0..58913d0b 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='2.5.110-01082024' +VERS='2.5.112-01092024' noheader=' update service dailytemp confsync -r -report ' diff --git a/inc/power.inc.sh b/inc/power.inc.sh index 072fbde5..ccc9d1ff 100644 --- a/inc/power.inc.sh +++ b/inc/power.inc.sh @@ -14,10 +14,17 @@ AVERAGEPOWER(){ } GET_OFFSITEHOST_FANSPEED(){ - vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) + vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') echo ${FanSpeed} > ${PW_TMPFOLDER}/.fanspeed } +GET_HOST_FANSPEED(){ + for ESXIHOST in "${PW_ALLHOSTS[@]}"; do + vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) + FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') + echo ${FanSpeed} > ${PW_TMPFOLDER}/.fanspeed.${ESXIHOST} + done +} CHECKPOWER(){ start=`date +%s` @@ -42,7 +49,7 @@ CHECKPOWER(){ done [ "${avgdays}" == "0" ] && unset avgdays [ "${avgdays}" != "" ] && avgdayshdr="${avgdays}-Day-Avg" || avgdayshdr="" - [ "${PTYPE}" != "power" ] && GET_OFFSITEHOST_FANSPEED & + [ "${PTYPE}" != "power" ] && GET_HOST_FANSPEED & if ([ "${STYPE}" == "" ] || [ "${STYPE}" == "mainsite" ]) && [ "${PTYPE}" != "esxi" ]; then [ ${REPORT} -eq 0 ] && echo @@ -212,7 +219,7 @@ CHECKPOWER(){ OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` OFFSITE_BATT_CAP=$(echo "${upsinfo}" | grep 'battery.charge:' | sed 's/.*: //') - # vHOSTFAN=$(ssh root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) + # vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r 10.2.1.21 -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan1" 2>&1) # FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $8}' | sed -e 's/%//g') # OFFSITE_VOLTIN=$(snmpwalk -t 1 -r 0 -v 1 -c public 10.2.1.100 iso.3.6.1.4.1.3808.1.1.1.3.2.1.0 | sed 's/.*: //') @@ -323,17 +330,17 @@ CHECKPOWER(){ average=$(AVERAGEPOWER ${PW_POWERSENSOR_ID[${SENSOR}]} ${avgdays}); average_disp="${idsCL[LightYellow]} `IDS_NUMBER_FORMAT ${average}`'watts" echo -en "\r\033[K"; echo -en "${MSG}${idsCL[LightYellow]}${average_disp}" else - if [ "${PW_HOST_IDRACS[${SENSOR}]}" == "10.2.1.21" ]; then + # if [ "${PW_HOST_IDRACS[${SENSOR}]}" == "10.2.1.21" ]; then # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSOR}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) # FanSpeed=$(ROUND_NUMBER `echo "scale=0; $(ROUND_NUMBER `echo "scale=4; ${vHOSTFAN}/21200" | bc` 2)*100/1" | bc` 0) echo -en "${idsCL[LightCyan]} Getting FanSpeed..." - until [ -f ${PW_TMPFOLDER}/.fanspeed ]; do tmp=tmp; done - echo -en "\r\033[K${MSG}${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]${idsCL[LightCyan]} [FanSpeed=$(cat ${PW_TMPFOLDER}/.fanspeed)'%]" - rm -f ${PW_TMPFOLDER}/.fanspeed + until [ -f ${PW_TMPFOLDER}/.fanspeed.${PW_ESXI_HOST_NAMES[${SENSOR}]} ]; do tmp=tmp; done + echo -en "\r\033[K${MSG}${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]${idsCL[LightCyan]} [FanSpeed=$(cat ${PW_TMPFOLDER}/.fanspeed.${PW_ESXI_HOST_NAMES[${SENSOR}]})'%]" + rm -f ${PW_TMPFOLDER}/.fanspeed.${PW_ESXI_HOST_NAMES[${SENSOR}]} - else - echo -en "${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]" - fi + # else + # echo -en "${idsCL[Cyan]} [PS1=${idsCL[LightCyan]}${PS1}'A${idsCL[Cyan]}, PS2=${idsCL[LightCyan]}${PS2}'A${idsCL[Cyan]}]" + # fi fi fi echo -e "${idsCL[Default]}" @@ -534,7 +541,7 @@ Shutting down main servers" 1 ######offsite check # if [ "$(CHECK_HOST 10.2.1.1)" != "false" ]; then -# upsinfo=$(ssh admin@10.2.1.1 'upsc CP1500PFCLCD') +# upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') # OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') # OFFSITE_BATT_RUNTIME=$(echo "${upsinfo}" | grep 'battery.runtime:' | sed 's/.*: //') # OFFSITE_BATT_RUNTIME=`echo "scale=2; ${OFFSITE_BATT_RUNTIME}/60" | bc` diff --git a/offsite-power-check.sh b/offsite-power-check.sh index e244f274..1fa94d9b 100755 --- a/offsite-power-check.sh +++ b/offsite-power-check.sh @@ -86,9 +86,9 @@ $(date)" for host in ${PW_OFFSITEHOSTS_IDRAC}; do if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) else echo "TESTMODE: Powering up '${host}'" fi @@ -115,10 +115,10 @@ $(date)" touch ${PW_TMPFOLDER}/shutdown.OFFSITE for host in ${PW_OFFSITEHOSTS}; do if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - ssh root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & fi done fi diff --git a/powerwall.sh b/powerwall.sh index a2bffce5..93ac88e2 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -136,13 +136,13 @@ SHUTDOWN_CRIT(){ echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPi System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - OctoPi System shutting down" >> ${PW_LOGFILE} sleep 1s - ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff elif [ "${1}" == "OctoPi-CPU-Usage" ]; then SENDNOTICE "${1} USAGE CRITICAL" "OctoPi System rebooting" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL USAGE - OctoPi System rebooting" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} USAGE CRITICAL - OctoPi System rebooting" >> ${PW_LOGFILE} - ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot fi } @@ -160,10 +160,10 @@ MAINT_MODE_VERIFY_SHUTDOWN(){ MaintMode="" until [ "${MaintMode}" == "Enabled" ]; do sleep 3s - MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get') + MaintMode=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get') done - ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' - ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' } @@ -271,7 +271,7 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) echo -e "${idsCL[Green]}Done${idsCL[Default]}" touch ${PW_TMPFOLDER}/shutdown.cls.offsite else @@ -283,7 +283,7 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) echo -e "${idsCL[Green]}Done${idsCL[Default]}" touch ${PW_TMPFOLDER}/shutdown.cls.mainsite else @@ -294,7 +294,7 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & + ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" @@ -342,7 +342,7 @@ SHUTDOWN_SERVER(){ echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do if [ "${TEST}" != "true" ]; then - if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" "${ESXIHOST}" & else touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done