From e6016adc5763cabc3c7a3a3014d48daf3dde9d36 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Fri, 26 Jan 2024 14:47:06 -0600 Subject: [PATCH] update --- inc/power.inc.sh | 4 ++-- inc/temp.inc.sh | 16 ++++++++-------- offsite-power-check.sh | 14 +++++++------- powerwall.sh | 18 +++++++++--------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/inc/power.inc.sh b/inc/power.inc.sh index fa1f5a8b..6465f9d3 100644 --- a/inc/power.inc.sh +++ b/inc/power.inc.sh @@ -214,7 +214,7 @@ CHECKPOWER(){ [ ${REPORT} -eq 0 ] && echo -en "${idsCL[Yellow]}Pulling data from 'Offsite' ... " - upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') + upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then OFFSITE_VOLTIN=$(echo "${upsinfo}" | grep 'input.voltage:' | sed 's/.*: //') OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //') @@ -613,7 +613,7 @@ POWERLOGGER_SERVICE(){ ${mysql_conn} -e "USE servermonitor; ${QRY}" fi - upsinfo=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') + upsinfo=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 admin@10.2.1.1 'upsc CP1500PFCLCD') if [ "$(echo "${upsinfo}" | grep 'input.voltage:')" != "" ]; then OFFSITE_VOLTOUT=$(echo "${upsinfo}" | grep 'output.voltage:' | sed 's/.*: //') OFFSITE_LOAD=$(echo "${upsinfo}" | grep 'ups.load:' | sed 's/.*: //') diff --git a/inc/temp.inc.sh b/inc/temp.inc.sh index a7b126fd..7e47c601 100644 --- a/inc/temp.inc.sh +++ b/inc/temp.inc.sh @@ -1,5 +1,5 @@ GET_HOST_FANSPEED(){ - vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${1}]}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) + vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${1}]}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then FanSpeed=$(echo ${vHOSTFAN} | awk '/ / {print $9}' | sed -e 's/%//g') else @@ -196,7 +196,7 @@ CHECKTEMPSENSOR(){ temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${1}]}" == "remotesystem" ]; then - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp) + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} vcgencmd measure_temp) if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} @@ -210,7 +210,7 @@ CHECKTEMPSENSOR(){ if [ "${1}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage") + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi fi @@ -394,7 +394,7 @@ CHECKTEMP_SERVICE_RUNSENSOR(){ ####### Fan Issue Workaround ####### if [ ! -f ${PW_TMPFOLDER}/${SENSORa}.down ] && ([ ! -f ${PW_TMPFOLDER}/.lastfancheck/${SENSORa} ] || [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/.lastfancheck/${SENSORa})) -ge 300 ]); then # vHOSTFAN=$(snmpwalk -v 1 -c public -t 2 ${PW_HOST_IDRACS[${SENSORa}]} .1.3.6.1.4.1.674.10892.5.4.700.12.1.6.1.3 -O vq) - vHOSTFAN=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) + vHOSTFAN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Fan3" 2>&1) if [ "${vHOSTFAN}" != "" ]; then # FanSpeed=`echo "scale=2; ${vHOSTFAN}/21200" | bc`; FanSpeed=$(ROUND_NUMBER `echo "scale=0; ${FanSpeed}*100" | bc | sed -e 's/.00//g'` 0) if [ "$(echo ${vHOSTFAN} | awk '/ / {print $4}' | sed -e 's/%//g')" == "RPM" ]; then @@ -445,7 +445,7 @@ CHECKTEMP_SERVICE_RUNSENSOR(){ fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSOR}]} vcgencmd measure_temp) temp_c=${temp_c%\'*}; temp_c=${temp_c#*=} temp_h='' @@ -468,7 +468,7 @@ CHECKTEMP_SERVICE_RUNSENSOR(){ if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi temp_h='' @@ -786,7 +786,7 @@ SENDTEMP(){ temp_c=$(cat /sys/class/hwmon/hwmon2/fan1_input) fi elif [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "remotesystem" ]; then - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} vcgencmd measure_temp) + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} vcgencmd measure_temp) if [ "${temp_c}" != "" ]; then temp_c=${temp_c%\'*} temp_c=${temp_c#*=} @@ -800,7 +800,7 @@ SENDTEMP(){ if [ "${SENSORa}" == "Powerwall-CPU-Usage" ]; then temp_c=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g') else - temp_c=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") + temp_c=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${SENSORa}]} "/opt/idssys/defaults/get-data.sh cpu-usage") fi else temp_c='' diff --git a/offsite-power-check.sh b/offsite-power-check.sh index cbeeccfb..d2ec6c90 100755 --- a/offsite-power-check.sh +++ b/offsite-power-check.sh @@ -88,9 +88,9 @@ $(date)" for host in ${PW_OFFSITEHOSTS_IDRAC}; do if [ "${TEST}" == "" ] || [ "${TEST}" == "full" ]; then ipmitool -I lanplus -H ${host} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power on & - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/True<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) else echo "TESTMODE: Powering up '${host}'" fi @@ -118,10 +118,10 @@ $(date)" touch ${PW_TMPFOLDER}/shutdown.OFFSITE for host in ${PW_OFFSITEHOSTS}; do if [ "${TEST}" = "" ] || [ "${TEST}" = "full" ]; then - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${host} sh < ${PW_FOLDER}/esxi-scripts/esxi-shutdown2.sh & fi done fi diff --git a/powerwall.sh b/powerwall.sh index 4ccdc63b..1f60a425 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -136,13 +136,13 @@ SHUTDOWN_CRIT(){ echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPi System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - OctoPi System shutting down" >> ${PW_LOGFILE} sleep 1s - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff elif [ "${1}" == "OctoPi-CPU-Usage" ]; then SENDNOTICE "${1} USAGE CRITICAL" "OctoPi System rebooting" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL USAGE - OctoPi System rebooting" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} USAGE CRITICAL - OctoPi System rebooting" >> ${PW_LOGFILE} - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot fi } @@ -165,11 +165,11 @@ MAINT_MODE_VERIFY_SHUTDOWN(){ MaintMode="" until [ "${MaintMode}" == "Enabled" ]; do sleep 3s - MaintMode=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get') + MaintMode=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get') done - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' sleep 4s - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.disable ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.down } @@ -292,7 +292,7 @@ SHUTDOWN_SERVER(){ # c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done # echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " # if [ "${TEST}" != "true" ]; then - # run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + # run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) # echo -e "${idsCL[Green]}Done${idsCL[Default]}" # touch ${PW_TMPFOLDER}/shutdown.cls.offsite # else @@ -304,7 +304,7 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - run=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) echo -e "${idsCL[Green]}Done${idsCL[Default]}" touch ${PW_TMPFOLDER}/shutdown.cls.mainsite else @@ -315,7 +315,7 @@ SHUTDOWN_SERVER(){ c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ "${TEST}" != "true" ]; then - ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & + ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" @@ -362,7 +362,7 @@ SHUTDOWN_SERVER(){ echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do if [ "${TEST}" != "true" ]; then - if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then + if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" "${ESXIHOST}" & else touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done