This commit is contained in:
2023-12-20 08:44:43 -06:00
parent 509474d24a
commit 62b4d3976d
3 changed files with 74 additions and 51 deletions

View File

@@ -245,15 +245,24 @@ SHUTDOWN_SERVER(){
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" == "" ]; then
hostonline=0
else
hostonline=1
fi
if [ ${hostonline} -eq 1 ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
fi
sleep 5s
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
sleep 5s
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
@@ -286,7 +295,6 @@ SHUTDOWN_SERVER(){
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
MSG="Restarting vCenter CLS Services"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
@@ -304,26 +312,33 @@ SHUTDOWN_SERVER(){
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
sleep 30s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
if [ ${MM} -eq 0 ] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then
tmp=tmp
else
if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
sleep 35s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
echo
fi
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then
@@ -332,10 +347,10 @@ SHUTDOWN_SERVER(){
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
if [ "${TEST}" != "true" ]; then
if [ "$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then
VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" "${ESXIHOST}" &
else
touch ${PW_TMPFOLDER}/verify.iscsi.${ESXI_HOST}.done
touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done
fi
fi
done
@@ -344,9 +359,9 @@ SHUTDOWN_SERVER(){
nc_count=0
until [ ${nc_count} -eq ${#PW_MAINSITEHOSTS[@]} ]; do
for iscsistatus in ${PW_TMPFOLDER}/verify.iscsi.*.done; do
NTS=$(grep -oP '(?<=iscsi.).*?(?=.done)' <<< "${iscsistatus}")
if [ "${NTS}" != "*" ]; then
rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NTS}.done
NICC=$(grep -oP '(?<=iscsi.).*?(?=.done)' <<< "${iscsistatus}")
if [ "${NICC}" != "*" ]; then
rm -f ${STATUSRUN_TMPFOLDER}/status-check.${NICC}.done
((nc_count++))
fi
done
@@ -359,7 +374,7 @@ SHUTDOWN_SERVER(){
MSG="Shutting down iSCSI-Server(s)"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
@@ -379,9 +394,13 @@ SHUTDOWN_SERVER(){
MSG="Shutting down remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
if [ "${TEST}" != "true" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
@@ -393,21 +412,25 @@ SHUTDOWN_SERVER(){
MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
if [ "${TEST}" != "true" ]; then
if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Waiting for iDS-vMS-iSCSI.scity.us to shutdown"
if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//iDS-vMS-iSCSI.scity.us/${idsCL[LightCyan]}iDS-vMS-iSCSI.scity.us}${idsCL[White]}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
@@ -416,7 +439,7 @@ SHUTDOWN_SERVER(){
MSG="Powering off SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
@@ -424,7 +447,7 @@ SHUTDOWN_SERVER(){
fi
fi
[ "${ESXI_HOST}" != "10.10.2.10" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n"
[ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n"
touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST}
else
echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n"
@@ -439,13 +462,13 @@ SHUTDOWN_SERVER(){
VERIFY_ISCSI_VM_DOWN(){
if [ "${1}" == "10.10.2.10" ] || [ "${1}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${2} >/dev/null 2>&1
elif [ "${1}" == "10.10.2.17" ] || [ "${1}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${2} >/dev/null 2>&1
elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${2} ###>/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${2} >/dev/null 2>&1
fi
touch ${PW_TMPFOLDER}/verify.iscsi.${1}.done
touch ${PW_TMPFOLDER}/verify.iscsi.${2}.done
}
HOSTMGMT(){