From b9e033ddf81b001749dc0edf207491813c7cf903 Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sat, 9 Dec 2023 23:09:16 -0600
Subject: [PATCH] Update powerwall.sh

---
 powerwall.sh | 293 +++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 235 insertions(+), 58 deletions(-)

diff --git a/powerwall.sh b/powerwall.sh
index eea8877b..628e2c7c 100755
--- a/powerwall.sh
+++ b/powerwall.sh
@@ -1263,38 +1263,54 @@ SHUTDOWN_SERVERS(){
 		if [ ! -f ${PW_FOLDER}/shutdown.main ]; then
 			touch ${PW_FOLDER}/shutdown.main
 			
-			MSG="Disabling CLS in vCenter"
-			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
-			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-			if [ "${TEST}" != "true" ]; then
-				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
-				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
-				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
+			if [ "${2}" == "TEMP" ]; then
+				MSG="Disabling CLS in vCenter"
+				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+				if [ "${TEST}" != "true" ]; then
+					run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
+					run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+					run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			fi
-			echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
 		
 			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				MSG="Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
@@ -1306,55 +1322,100 @@ SHUTDOWN_SERVERS(){
 					run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
 				fi
 				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
 			done
 			echo
 			for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
 				MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
-				echo -e "${idsCL[Green]}Continuing in background"
+				if [ "${TEST}" != "true" ]; then
+					MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
+					echo -e "${idsCL[Green]}Continuing in background\n"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
+			MSG="Waiting for iSCSI-PRI Host to shutdown"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
+				echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+			MSG="Powering OFF the SAS Enclosure"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
+					run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
+					sleep 3s
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+				else
+					echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
+				fi
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+			echo
 		fi
 	
 	elif [ "${1^^}" == "SYS" ]; then
-		[ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN
+		[ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP
 		if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then
 			touch ${PW_FOLDER}/shutdown.sys
 			
-			MSG="Disabling CLS in vCenter"
-			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
-			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-			if [ "${TEST}" != "true" ]; then
-				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
-				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
-				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+			if [ "${2}" == "TEMP" ]; then
+				MSG="Disabling CLS in vCenter"
+				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+				if [ "${TEST}" != "true" ]; then
+					run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
+					run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+					run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+				fi
+				echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
 			fi
-			echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
 						
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				MSG="Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
@@ -1364,16 +1425,22 @@ SHUTDOWN_SERVERS(){
 				if [ "${TEST}" != "true" ]; then
 					${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
 					${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
 				fi
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
 			done
 			echo
 			for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
 				MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
-				echo -e "${idsCL[Green]}Continuing in background\n"
+				if [ "${TEST}" != "true" ]; then
+					MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
+					echo -e "${idsCL[Green]}Continuing in background\n"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 		fi
@@ -1386,6 +1453,7 @@ SHUTDOWN_SERVERS(){
 			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
 			if [ "${TEST}" != "true" ]; then
+				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
 				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
 				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
 				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
@@ -1396,24 +1464,36 @@ SHUTDOWN_SERVERS(){
 				MSG="Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
 				MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
 				MSG="Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
@@ -1423,16 +1503,22 @@ SHUTDOWN_SERVERS(){
 				if [ "${TEST}" != "true" ]; then
 					${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
 					${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
 				fi
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
 			done
 			echo
 			for ESXIHOST in "${PW_ALLHOSTS[@]}"; do		
 				MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
-				echo -e "${idsCL[Green]}Continuing in background"
+				if [ "${TEST}" != "true" ]; then
+					MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
+					echo -e "${idsCL[Green]}Continuing in background\n"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
 			MSG="Waiting for iSCSI-PRI Host to shutdown"
@@ -1461,46 +1547,137 @@ SHUTDOWN_SERVERS(){
 			echo
 		fi
 		
-	elif [ "${1^^}" == "OFFSITE" ]; then
-		if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
-			touch ${PW_FOLDER}/shutdown.offsite
+	elif [ "${1^^}" == "MAINSITE" ]; then
+		if [ ! -f ${PW_FOLDER}/shutdown.mainsite ]; then
+			touch ${PW_FOLDER}/shutdown.mainsite
 			
 			MSG="Disabling CLS in vCenter"
 			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
 			if [ "${TEST}" != "true" ]; then
-				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
+				run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
 				run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
 				run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
 			fi
-			echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
-			
-			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+		else
+			echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+		fi			
+			for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
 				MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
-			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
 				MSG="Stopping VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
-				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				if [ "${TEST}" != "true" ]; then
+					run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
-			for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
 				MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
 				c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
 				echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
-				[ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
-				echo -e "${idsCL[Green]}Continuing in background\n"
+				if [ "${TEST}" != "true" ]; then
+					MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
+					echo -e "${idsCL[Green]}Continuing in background\n"
+				else
+					echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+				fi
 			done
 			echo
+			MSG="Waiting for iSCSI-PRI Host to shutdown"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
+				echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+			MSG="Powering OFF the SAS Enclosure"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
+					run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
+					sleep 3s
+					echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+				else
+					echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
+				fi
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+			echo
 		fi
 	fi
+	
+elif [ "${1^^}" == "OFFSITE" ]; then
+	if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then
+		touch ${PW_FOLDER}/shutdown.offsite
+		
+		MSG="Disabling CLS in vCenter"
+		c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+		echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+		if [ "${TEST}" != "true" ]; then
+			run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
+			run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
+			run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
+		fi
+		echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
+		
+		for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
+				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+		done
+		echo
+		for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			MSG="Stopping VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
+				echo -e "${idsCL[Green]}Done${idsCL[Default]}"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+		done
+		echo
+		for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
+			MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
+			c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
+			echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
+			if [ "${TEST}" != "true" ]; then
+				MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
+				echo -e "${idsCL[Green]}Continuing in background\n"
+			else
+				echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
+			fi
+		done
+		echo
+	fi
+fi
 }
 MAINT_MODE_VERIFY_SHUTDOWN(){
 	ESXIHOST=${1}