update

2023-11-02 15:59:07 -05:00
parent a71fd622ce
commit 83b1ec4d68
2 changed files with 71 additions and 35 deletions
--- a/defaults.inc
+++ b/defaults.inc
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-VERS='2.3.120-11022023'
+VERS='2.3.130-11022023'

 noheader=' update service dailytemp '

@@ -121,11 +121,13 @@ PW_MAINSITE_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS}"
 PW_ALL_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS},${PW_OFFSITEHOSTS}"

 PW_RACADM_ACCESS=10.10.0.241
+RACADMONLINE=$(CHECK_HOST ${PW_RACADM_ACCESS})

 IFS=,
 PW_MAINSITE_HOSTS=(${PW_MAINSITE_HOSTS})
 PW_MAINHOSTS=(${PW_MAINHOSTS})
 PW_SYSHOSTS=(${PW_SYSHOSTS})
+PW_OFFSITEHOSTS=(${PW_OFFSITEHOSTS})
 PW_ALLHOSTS=(${PW_ALL_HOSTS})
 unset IFS

--- a/powerwall.sh
+++ b/powerwall.sh
@@ -34,7 +34,7 @@ CHECKTEMP(){
 				echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[LightCyan]}- ${vHOSTiDRACIP}${idsCL[Default]}"
 				[ "${1}" != "noavg" ] && echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..." || echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ..."
 				
-				if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then
+				if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ] && [ "${RACADMONLINE}" != "false" ]; then
 					vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
 					
 					# iDS-vMS-Offsite Fan Issue Workaround
@@ -68,7 +68,9 @@ CHECKTEMP(){
 					else
 						echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: No Data"
 					fi
-		
+				elif [ "${RACADMONLINE}" == "false" ]; then
+					echo -e "\r\033[K${idsCL[Red]}${SENSOR} RACADM is down"
+					
 				else #host down
 					echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down"
 					
@@ -213,7 +215,7 @@ CHECKTEMP_SERVICE(){
 				
 				fi
 					
-				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
+				if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ "${RACADMONLINE}" != "false" ]; then
 					unset IFS
 					SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
 					IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
@@ -233,6 +235,9 @@ CHECKTEMP_SERVICE(){
 						fi
 					fi
 					
+				elif [ "${RACADMONLINE}" == "false" ]; then
+					SENSORS_CHECK=''
+					
 				else
 					SENSORS_CHECK=(${SENSORa})
 				fi
@@ -451,10 +456,12 @@ DAILYTEMP(){
 	DIVIDER . . 60
 	for SENSORa in ${PW_SENSOR_ORD[@]}; do
 		
-		if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then
+		if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ "${RACADMONLINE}" != "false" ]; then
 			unset IFS
 			SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1)
 			IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS
+		elif [ "${RACADMONLINE}" == "false" ]; then
+			SENSORS_CHECK=''
 		else
 			SENSORS_CHECK=(${SENSORa})
 		fi
@@ -508,12 +515,14 @@ DAILYTEMP(){
 GETHOSTPOWER(){
 	hostip=${PW_iDRACHOST_SHORTNAMES[${1}]}
 	hostip=${PW_HOST_ADDRESSES[${hostip}]}
-	if [ "$(CHECK_HOST ${hostip})" != "false" ]; then
+	if [ "$(CHECK_HOST ${hostip})" != "false" ] && [ "${RACADMONLINE}" != "false" ]; then
 		if [ "${1}" == "10.10.2.26" ]; then
 			temp=0
 		else
 			temp=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${1} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Pwr"  | awk '/ / {print $6}' | sed -e 's/Watts//g' 2>&1)
 		fi
+	elif [ "${RACADMONLINE}" == "false" ]; then
+		temp=racadmdown
 	else
 		temp=down
 	fi
@@ -550,8 +559,8 @@ CHECKPOWER(){
 	fi

 	if [ "${1}" == "" ] || [ "${1}" == "sensors" ]; then
-		if [ "$(CHECK_HOST inv1.scity.us)" != "false" ]; then
-			INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json")
+		if [ "$(CHECK_HOST 10.10.0.61)" != "false" ]; then
+			INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json")
 			INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'`
 			INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'`
 			OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'`
@@ -1006,23 +1015,23 @@ SHUTDOWN_SERVERS(){
 			fi
 			echo "Done"
 		
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				echo -en "Stopping iSCSI-PRI VM's on: ${ESXIHOST} ... "
 				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
 				echo "Done"
 			done
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... "
 				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
 				echo "Done"
 			done
 			# sleep 1m
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				echo -en "Verifying iSCSI-PRI VM's stopped on: ${ESXIHOST} ... "
 				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
 				echo "Done"
 			done
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
 				if [ "${TEST}" != "true" ]; then
 					run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
@@ -1030,7 +1039,12 @@ SHUTDOWN_SERVERS(){
 				fi
 				echo "Done"
 			done
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
+				echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... "
+				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+				echo "Done"
+			done
+			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..."
 				if [ "${TEST}" != "true" ]; then
 					run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"')
@@ -1038,16 +1052,7 @@ SHUTDOWN_SERVERS(){
 				fi
 				echo "Done"
 			done
-		
-			# for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
-			# 	echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
-			# 	if [ "${TEST}" != "true" ]; then
-			# 	    while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
-			# 	    	sleep 1
-			# 	    done
-			# 	fi
-			# 	echo "Done"
-			# done
+			
 		fi
 	fi
 	if [ "${1^^}" == "SYS" ] || [ "${1^^}" == "ALL" ]; then
@@ -1076,16 +1081,11 @@ SHUTDOWN_SERVERS(){
 				echo "Done"
 			done
 			# sleep 1m
-			for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do
+			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				echo -en "Verifying iSCSI-SYS VM's stopped on: ${ESXIHOST} ... "
 				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
 				echo "Done"
 			done
-			# for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
-			# 	echo -en "Suspending MySQL & LAN-DNS VM's on: ${ESXIHOST} ... "
-			# 	[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1
-			# 	echo "Done"
-			# done
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
 				if [ "${TEST}" != "true" ]; then
@@ -1094,6 +1094,11 @@ SHUTDOWN_SERVERS(){
 				fi
 				echo "Done"
 			done
+			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
+				echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... "
+				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+				echo "Done"
+			done
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				echo -en "Shutting Down Host: ${ESXIHOST} ... "
 				if [ "${TEST}" != "true" ]; then
@@ -1102,16 +1107,45 @@ SHUTDOWN_SERVERS(){
 				fi
 				echo "Done"
 			done
-
-			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
-				echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
+			
+		fi
+	fi
+	
+	if [ "${1^^}" == "OFFSITE" ]; then
+		if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
+			touch ${PW_FOLDER}/shutdown.offsite
+			
+			echo -en "Disabling CLS in vCenter ... "
+			if [ "${TEST}" != "true" ]; then
+				# run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
+				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
+			fi
+			echo "Done"
+			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+				echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... "
+				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+				echo "Done"
+			done
+			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+				echo -en "Stopping VM's on: ${ESXIHOST} ... "
+				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+				echo "Done"
+			done
+			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+				echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... "
+				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+				echo "Done"
+			done
+			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+				echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..."
 				if [ "${TEST}" != "true" ]; then
-					while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
-						sleep 1
-					done
+					run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"')
+					run=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0')
 				fi
 				echo "Done"
 			done
+			
 		fi
 	fi
 }