voltron/powerwall
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-12-13 21:33:21 -06:00
parent be740584bf
commit 40e49f60bf
2 changed files with 239 additions and 591 deletions
Download Patch File Download Diff File Expand all files Collapse all files

828
powerwall.sh
View File

@@ -923,14 +923,14 @@ BATT VOLTAGE: ${BATTVOLT}'volts" 1
touch ${PW_TMPFOLDER}/power.ac.low
##### SHUTDOWN OFFSITE SERVER WHILE ITS AT MAINSITE
SHUTDOWN_SERVERS2 10.2.1.11 &
SHUTDOWN_SERVER 10.2.1.11 &
fi
if [ ! -f ${PW_TMPFOLDER}/power.ac.conserve ] && [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ac.low)) -ge $(echo "scale=0; ${PW_CONSERVE_AFTER}*60" | bc) ]; then
echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${INPUTACV}'volts / ${BATTVOLT}'volts - Shutting down iSCSI VM's and Host to conserve power" >> ${PW_LOGPOWER}
SENDNOTICE "POWER ALERT - Conserving Power" "Shutting down iSCSI VM's and Host to conserve power" 1
SHUTDOWN_SERVERS2 ${PW_HOST_IDRACS['iDS-vMS-Host0']} &
SHUTDOWN_SERVER ${PW_HOST_IDRACS['iDS-vMS-Host0']} &
touch ${PW_TMPFOLDER}/power.ac.conserve
fi
fi
@@ -1215,15 +1215,15 @@ CHECK_SERVICES(){
SHUTDOWN_MAIN(){
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then
# SHUTDOWN_SERVERS MAIN ${1}
SHUTDOWN_SERVER2 main
# SHUTDOWN_SERVER MAIN ${1}
SHUTDOWN_SERVER main
# HOSTMGMT main -e off >/dev/null 2>&1
reason='Server Room Overheated!'
sendnotice=true
elif [ "${1}" == "POWER" ]; then
# SHUTDOWN_SERVERS MAIN ${1}
SHUTDOWN_SERVER2 main
# SHUTDOWN_SERVER MAIN ${1}
SHUTDOWN_SERVER main
# HOSTMGMT main -e off >/dev/null 2>&1
reason='Low Power!'
sendnotice=true
@@ -1242,15 +1242,15 @@ ${reason}" 1
}
SHUTDOWN_SYS(){
if [ "${1}" == "ServerRoomTH" ] || [ "${1}" == "TEMP" ]; then
# SHUTDOWN_SERVERS SYS ${1}
[ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER2 mainsite || SHUTDOWN_SERVER2 sys
# SHUTDOWN_SERVER SYS ${1}
[ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER mainsite || SHUTDOWN_SERVER sys
# HOSTMGMT sys -e off >/dev/null 2>&1
reason='Server Room Overheated!'
sendnotice=true
elif [ "${1}" == "POWER" ]; then
# SHUTDOWN_SERVERS SYS ${1}
[ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER2 mainsite || SHUTDOWN_SERVER2 sys
# SHUTDOWN_SERVER SYS ${1}
[ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ] && SHUTDOWN_SERVER mainsite || SHUTDOWN_SERVER sys
# HOSTMGMT sys -e off >/dev/null 2>&1
reason='Low Power!'
sendnotice=true
@@ -1291,490 +1291,16 @@ SHUTDOWN_CRIT(){
ssh root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot
fi
}
SHUTDOWN_OFFSITE(){
if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then
SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1
for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do
SHUTDOWN_SERVER ${offsite_host} &
done
touch ${PW_TMPFOLDER}/power.offsite.off
fi
}
SHUTDOWN_SERVERS(){
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${1} servers" >> ${PW_LOGFILE}
[ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE}
if [ "${1^^}" == "MAIN" ]; then
if [ ! -f ${PW_TMPFOLDER}/power.main.off ]; then
touch ${PW_TMPFOLDER}/power.main.off
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
MSG="Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${PW_MAINHOSTS[@]}"; do
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
MSG="Waiting for iSCSI-PRI Host to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering OFF the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
fi
elif [ "${1^^}" == "SYS" ]; then
[ ! -f ${PW_TMPFOLDER}/power.main.off ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN TEMP
if [ ! -f ${PW_TMPFOLDER}/power.sys.off ]; then
touch ${PW_TMPFOLDER}/power.sys.off
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
MSG="Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${PW_SYSHOSTS[@]}"; do
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
fi
fi
elif [ "${1^^}" == "ALL" ]; then
if [ ! -f ${PW_TMPFOLDER}/power.all.off ]; then
touch ${PW_TMPFOLDER}/power.all.off
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
MSG="Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${PW_ALLHOSTS[@]}"; do
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
MSG="Waiting for iSCSI-PRI Host to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering OFF the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
fi
elif [ "${1^^}" == "MAINSITE" ]; then
if [ ! -f ${PW_TMPFOLDER}/power.main.offsite ]; then
touch ${PW_TMPFOLDER}/power.main.offsite
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Verifying iSCSI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
MSG="Waiting for iSCSI-PRI Host to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering OFF the SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2 >/dev/null 2>&1)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[LightRed]}'${PW_iDRACHOST_SHORTNAMES[${idracip}]}' is not offline, not powering down\n"
fi
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
fi
elif [ "${1^^}" == "OFFSITE" ]; then
if [ ! -f ${PW_TMPFOLDER}/power.offsite.off ]; then
touch ${PW_TMPFOLDER}/power.offsite.off
if [ "${2}" != "maintenance" ]; then
MSG="Disabling CLS in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1)
run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
MSG="Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
MSG="Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do
MSG="Wait for VM's to power off, then shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}'"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
fi
fi
fi
}
MAINT_MODE_VERIFY_SHUTDOWN(){
ESXIHOST=${1}
MaintMode=""
@@ -1782,139 +1308,261 @@ MAINT_MODE_VERIFY_SHUTDOWN(){
sleep 3s
MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get')
done
# ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"'
# ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"'
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
}
SHUTDOWN_SERVER(){
if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then
tmp=${PW_iDRACHOST_SHORTNAMES[${1}]}
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
elif [ "${PW_ESXI_HOST_NAMES[${1}]}" != "" ]; then
ESXIHOST=${1}
elif [ "${PW_HOST_IDRACS[iDS-vMS-Host${1}]}" != "" ]; then
ESXIHOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${1}]}
elif [ "${PW_HOST_IDRACS[${1}]}" != "" ]; then
tmp=${PW_HOST_IDRACS[${1}]}
ESXIHOST=${PW_HOST_ADDRESSES[${tmp}]}
else
ESXIHOST=''
if [ "${1}" != "-h" ]; then
HOST=${1}; MN=0
if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then
tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]}
ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]}
elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then
ESXI_HOST=${HOST}
elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}" != "" ]; then
ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}
elif ([ "${HOST::1}" == "o" ] || [ "${HOST::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}" != "" ]; then
ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}
elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then
ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]}
elif [ "${1^^}" == "ALL" ] || [ "${1^^}" == "MAINSITE" ] || [ "${1^^}" == "OFFSITE" ] || [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "SYS" ]; then
ESXI_HOST=${HOST^^}
MN=1
else
ESXI_HOST=''
fi
[ "${ESXI_HOST}" == "" ] && echo -e "Invalid host\n" && exit 1
if [ "${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}" != "" ]; then
tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}
idracip=${PW_HOST_IDRACS[${tmp}]}
hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]}
fi
fi
while [ $# -gt 0 ]; do
case "$1" in
-p|-power) POWER=${2}; MODE="power";;
-e|-esxi) ESXI=${2}; MODE="esxi";;
-h)
echo -e "Usage: ${idsCL[Yellow]}pw host {host} {flags}${idsCL[Default]} {"
width=35
printf "%-${width}s- %s\n" " {host}" "(*required, can either be esxiIP,idracIP or Host#)"
printf "%-${width}s- %s\n" " -p|-power {on / off}" "Controls Host Power"
printf "%-${width}s- %s\n" " -e|-esxi {off / mm}" "Shuts down ESXi or puts into maintenance mode"
echo -e "}\n"
exit 0
;;
esac
shift 1
done
if [ "${ESXIHOST}" != "" ]; then
host=${PW_ESXI_HOST_NAMES[${ESXIHOST}]}
if [ ${MN} -eq 1 ]; then
ESXI_HOSTS=PW_${ESXI_HOST}HOSTS[@]
ESXI_HOSTS=(${!ESXI_HOSTS})
[ "${ESXI_HOST}" == "ALL" ] && hostname="All Host Servers" || hostname="All ${ESXI_HOST} Host Servers"
else
ESXI_HOSTS=(${ESXI_HOST})
hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}
fi
echo
DIVIDER . yellow 75
echo -e "${idsCL[Yellow]}Shutting down - ${hostname}${idsCL[Default]}"
DIVIDER . yellow 75
if [ "${ESXI_HOSTS}" != "" ]; then
if [ "${2}" == "maintenance" ]; then
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${host} [${ESXIHOST}] into maintenance mode" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE}
else
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${host} [${ESXIHOST}]" >> ${PW_LOGFILE}
echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE}
fi
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-PRI host server, all iSCSI-PRI VM's are being shutdown ... "
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
if [ "${2}" != "maintenance" ] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ]); then
if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then
MSG="Disabling CLS for the Offsite Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
sleep 5s
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c105105>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
touch ${PW_TMPFOLDER}/shutdown.cls.offsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
[ "${TEST}" != "true" ] && sleep 10s
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
echo -e "${idsCL[LightYellow]}Since this is the iSCSI-SYS host server, all iSCSI-SYS VM's are being shutdown ... "
for ehost in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ehost}]} [${ehost}]"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ehost} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
sleep 5s
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
[ "${TEST}" != "true" ] && sleep 10s
fi
MSG="Enabling maintenance mode & starting iSCSI VM migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 25s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Shutting down remaining VM's"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
if [ "${ESXIHOST}" == "10.10.2.10" ] || [ "${ESXIHOST}" == "10.10.2.17" ]; then
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
elif [ "${ESXIHOST}" == "10.10.2.17" ]; then
run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
fi
run=$(${PW_FOLDER}/esxi-scripts/cls-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
if [ "${2}" != "maintenance" ]; then
MSG="Waiting for VM's to power off or migrate"
if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ]; then
MSG="Disabling CLS for the Primary Cluster in vCenter"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
run=$(ssh root@${PW_VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1)
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
touch ${PW_TMPFOLDER}/shutdown.cls.mainsite
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
fi
MSG="Restarting vCenter CLS Services"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background\n"
ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
if [ "${ESXIHOST}" == "10.10.2.10" ]; then
MSG="Waiting for Host ${host} [${ESXIHOST}] to shutdown"
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
MSG="Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
MSG="Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Stopping All iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
fi
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
fi
sleep 5s
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
fi
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 &
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" != "OFFSITE" ] && [ "${ESXI_HOST}" != "ALL" ] && [ "${ESXI_HOST}" != "MAINSITE" ]; then
MSG="Starting iSCSI migrations"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
sleep 20s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ]; then
for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do
MSG="Waiting for iSCSI VM's to power down on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
fi
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
MSG="Shutting down iSCSI-Server(s)"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "10.10.2.17" ] || [ "${ESXI_HOST}" == "SYS" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.17 >/dev/null 2>&1 &
elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.10 >/dev/null 2>&1 &
${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 10.10.2.17 >/dev/null 2>&1 &
fi
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
echo
fi
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Shutting down remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 &
echo -e "${idsCL[Green]}Commands sent${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${2}" != "maintenance" ]; then
for ESXIHOST in "${ESXI_HOSTS[@]}"; do
MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} &
echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
done
echo
if [ "${ESXI_HOST}" == "10.10.2.10" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then
MSG="Waiting for iDS-vMS-iSCSI.scity.us to shutdown"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[White]}${MSG//iDS-vMS-iSCSI.scity.us/${idsCL[LightCyan]}iDS-vMS-iSCSI.scity.us}${idsCL[White]}${MSG_SPC}: "
if [ "${TEST}" != "true" ]; then
while ping -qw 10 -c3 "10.10.2.10">/dev/null 2>&1; do sleep 1; done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
MSG="Powering off SAS Enclosure"
c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done
echo -en "${idsCL[LightCyan]}${MSG}${MSG_SPC}: "
echo -en "${idsCL[White]}${MSG}${MSG_SPC}: "
if [ "${TEST}" != "true" ] && [ "$(CHECK_HOST 10.10.2.10)" == "false" ]; then
run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2)
sleep 3s
echo -e "${idsCL[Green]}Done${idsCL[Default]}\n"
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}"
fi
else
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be shutdown momentarily ... \n"
fi
[ "${ESXI_HOST}" != "10.10.2.10" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n"
touch ${PW_TMPFOLDER}/shutdown.${ESXI_HOST}
else
echo -e "\n${idsCL[LightCyan]}The host ${host} [${ESXIHOST}] will be in maintenance mode momentarily ... \n"
echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n"
touch ${PW_TMPFOLDER}/maintmode.${ESXI_HOST}
fi
else
@@ -2005,23 +1653,23 @@ HOSTMGMT(){
if [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ]; then
if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then
echo -e "${idsCL[LightGreen]}PUTTING '${ESXI_HOST}' HOSTS INTO MAINTENANCE MODE${idsCL[Default]}\n"
# SHUTDOWN_SERVERS ${ESXI_HOST,,} maintenance
SHUTDOWN_SERVERS2 ${ESXI_HOST,,} maintenance
# SHUTDOWN_SERVER ${ESXI_HOST,,} maintenance
SHUTDOWN_SERVER ${ESXI_HOST,,} maintenance
else
echo -e "${idsCL[LightGreen]}SHUTTING DOWN '${ESXI_HOST}' HOSTS${idsCL[Default]}\n"
# SHUTDOWN_SERVERS ${ESXI_HOST,,}
SHUTDOWN_SERVERS2 ${ESXI_HOST,,}
# SHUTDOWN_SERVER ${ESXI_HOST,,}
SHUTDOWN_SERVER ${ESXI_HOST,,}
fi
else
if [ "${ESXI}" == "maint" ] || [ "${ESXI}" == "maintenance" ] || [ "${ESXI}" == "mm" ]; then
echo -e "${idsCL[LightGreen]}PUTTING HOST INTO MAINTENANCE MODE${idsCL[Default]}\n"
# SHUTDOWN_SERVER ${HOST} maintenance
SHUTDOWN_SERVER2 ${HOST} maintenance
SHUTDOWN_SERVER ${HOST} maintenance
else
echo -e "${idsCL[LightGreen]}SHUTTING DOWN HOST${idsCL[Default]}\n"
# SHUTDOWN_SERVER ${HOST}
SHUTDOWN_SERVER2 ${HOST}
SHUTDOWN_SERVER ${HOST}
fi
fi
@@ -2365,8 +2013,8 @@ fi
[ "${2^^}" == "SYS" ] && ISCSISYSVMSHUTDOWN || ISCSIVMSHUTDOWN
;;
shutdownhost) SHUTDOWN_SERVER ${2} ${3};;
shutdowntest) SHUTDOWN_SERVER2 ${2} ${3};;
shutdown_servers) SHUTDOWN_SERVERS ${2} ${3};;
shutdowntest) SHUTDOWN_SERVER ${2} ${3};;
shutdown_servers) SHUTDOWN_SERVER ${2} ${3};;
hostpower) HOSTPOWER ${2} ${3};;
host) HOSTMGMT ${2} ${3} ${4} ${5} ${6} ${7};;

2
shutdown-test.sh
View File

@@ -1,4 +1,4 @@
SHUTDOWN_SERVER2(){
SHUTDOWN_SERVER(){
if [ "${1}" != "-h" ]; then
HOST=${1}; MN=0
if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then