voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-12 21:51:02 -06:00
parent 15989c57a5
commit 8f55589ceb
3 changed files with 209 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
shutdown-test.sh
View File

@@ -1,6 +1,6 @@
SHUTDOWN_SERVER2(){
if [ "${1}" != "-h" ]; then
HOST=${1}
HOST=${1}; MN=0
if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then
tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]}
ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]}
@@ -13,6 +13,7 @@ SHUTDOWN_SERVER2(){
elif [ "${1^^}" == "ALL" ] || [ "${1^^}" == "MAINSITE" ] || [ "${1^^}" == "OFFSITE" ] || [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "SYS" ]; then
ESXI_HOST=${HOST^^}
hostname="All ${1^^} Host Servers"
MN=1
else
ESXI_HOST=''
fi
@@ -41,133 +42,207 @@ SHUTDOWN_SERVER2(){
shift 1
done
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
if [ ${MN} -eq 1 ]; then
ESXI_HOSTS=PW_${HOST^^}HOSTS[@]
ESXI_HOSTS=(${!ESXI_HOSTS})
else
ESXI_HOSTS=(${ESXI_HOST})
hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}
fi
for vhost in "${ESXI_HOSTS[@]}"; do
echo -e "HERE: ${idsCL[LightCyan]} ${PW_ESXI_HOST_NAMES[${vhost}]} [${vhost}]${idsCL[Default]}"
done
ESXIHOST=""
if [ "${ESXIHOST}" != "" ]; then
host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]}
if [ "${ESXI_HOSTS}" != "" ]; then
if [ "${2}" == "maintenance" ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${host} [${ESXIHOST}] into maintenance mode" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE}
else
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} [${ESXIHOST}]" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE}
fi
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
if [ "${2}" != "maintenance" ] && [ ${MN} -eq 1 ]; then
if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then
MSG="Disabling CLS for the Offsite Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
sleep 5s
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
touch ${PW_TMPFOLDER}/shutdown.cls.offsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
[ "${TEST}" != "true" ] && sleep 10s
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... "
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
sleep 5s
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
[ "${TEST}" != "true" ] && sleep 10s
fi
MSG="Enabling maintenance mode & starting iSCSI VM migrations"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 25s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Shutting down remaining VM's"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
if [ "${2}" != "maintenance" ]; then
MSG="Waiting for VM's to power off or migrate"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
if [ "${ESXI_HOST}" != "OFFSITE" ]; then
MSG="Disabling CLS for the Primary Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
touch ${PW_TMPFOLDER}/shutdown.cls.mainsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
MSG="Restarting vCenter CLS Services"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null & 2>&1 && service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null & 2>&1 &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
MSG="Waiting for Host ${host} [${ESXIHOST}] to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Stopping All iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
fi
sleep 5s
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
done
echo
[ "${TEST}" != "true" ] && sleep 10s
else
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Enabling maintenance mode & starting iSCSI migrations on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
sleep 25s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Shutting down remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Waiting for iSCSI VM's to be: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 10.10.2.10 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 10.10.2.17 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 10.10.2.10 >/dev/null 2>&1 &
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 10.10.2.17 >/dev/null 2>&1 &
fi
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
fi
MSG="Shutting down iSCSi-Server(s)"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
fi
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} [${ESXIHOST}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Waiting for iDS-vMS-iSCSI.scity.us to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering off SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${hc_cw} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
else
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be shutdown momentarily ... \n"
fi
[ "${ESXI_HOST}" != "10.10.2.10" ] && echo -e "\n${idsCL[LightCyan]}${hostname} will be shutdown momentarily ... \n"
else
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be in maintenance mode momentarily ... \n"
echo -e "\n${idsCL[LightCyan]}${hostname} will be in maintenance mode momentarily ... \n"
fi
else