diff --git a/defaults.inc b/defaults.inc index 5ddfa4ce..ad212b6f 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='2.3.152-11042023' +VERS='2.3.160-11042023' noheader=' update service dailytemp ' diff --git a/esxi-scripts/iscsi-all-vm-shutdown.ps1 b/esxi-scripts/iscsi-all-vm-shutdown.ps1 new file mode 100755 index 00000000..101d0675 --- /dev/null +++ b/esxi-scripts/iscsi-all-vm-shutdown.ps1 @@ -0,0 +1,9 @@ +#!/usr/bin/env pwsh + +. /opt/idssys/powerwall/settings.ps1 + +Connect-VIServer -Server $args[0] -Protocol https -User $ESXiUsername -Password $ESXiPassword | Out-Null + +Get-Datastore | Where {$_.Name -like '*iSCSI-*'} | Get-VM | Where {$_.PowerState -like '*On*' -and $_.Name -notlike '*CLS*'} | Shutdown-VMGuest -Confirm:$false + +exit 0 diff --git a/powerwall.sh b/powerwall.sh index 7ea66248..30a302ed 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -628,12 +628,26 @@ CHECKPOWER_SERVICE(){ voltstatus=0 battstatus=0 while true; do + datetime=`date +'%Y-%m-%d %H:%M:%S'` INVSTATSINFO=$(curl -s "http://${PW_INVERTER_IP}/stats.json") - if [ "${INVSTATSINFO}" != "" ]; then + SVRRM_UPS_INPUTACV=$(snmpwalk -t 1 -r 0 -v 1 -c public ${PW_UPS_IP} iso.3.6.1.4.1.850.1.1.3.1.3.2.2.1.3.1.1 | sed 's/.*: //') + + if (( $(bc <<<"${SVRRM_UPS_INPUTACV} < ${min_acvolt}") )); then + if [ ! -f ${PW_TMPFOLDER}/power.ups.low ]; then + touch ${PW_TMPFOLDER}/power.ups.low + + elif [ $(expr `date +%s` - $(stat -c %Y ${PW_TMPFOLDER}/power.ups.low)) -ge 60 ]; then + SENDNOTICE "POWER ALERT - UPS POWER OFF/LOW POWER!!" "(${datetime}) UPS has no power for 1min, shutting all servers down!" 1 + + fi + elif [ -f ${PW_TMPFOLDER}/power.ups.low ]; then + rm -f ${PW_TMPFOLDER}/power.ups.low + fi + + if [ "${INVSTATSINFO}" != "" ]; then INPUTACV=$(echo ${INVSTATSINFO} | jq '.inputs .inV') BATTVOLT=$(echo ${INVSTATSINFO} | jq '.inputs .battV') - datetime=`date +'%Y-%m-%d %H:%M:%S'` if (( $(bc <<<"${INPUTACV} >= ${min_acvolt}") )); then if [ ${voltstatus} -eq 3 ]; then @@ -969,7 +983,7 @@ SHUTDOWN_SERVERS(){ echo "(${datetime}) - Shutting down ${1} servers" >> ${PW_LOGFILE} [ "${TEST}" == "true" ] && echo "RUNNING IN TEST MODE">>${PW_LOGFILE} - if [ "${1^^}" == "MAIN" ] || [ "${1^^}" == "ALL" ]; then + if [ "${1^^}" == "MAIN" ]; then if [ ! -f ${PW_FOLDER}/shutdown.main ]; then touch ${PW_FOLDER}/shutdown.main @@ -982,46 +996,39 @@ SHUTDOWN_SERVERS(){ echo "Done" for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Stopping iSCSI-PRI VM's on: ${ESXIHOST} ... " + echo -en "Stopping iSCSI-PRI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... " + echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) echo "Done" done # sleep 1m for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Verifying iSCSI-PRI VM's stopped on: ${ESXIHOST} ... " + echo -en "Verifying iSCSI-PRI VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... " + echo -en "Stopping Remaining VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " if [ "${TEST}" != "true" ]; then run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) run=$(${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) fi echo "Done" done + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " - [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo "Done" - done - for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..." - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"') - run=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0') - fi - echo "Done" + echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... " + [ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background\n" done fi - fi - if [ "${1^^}" == "SYS" ] || [ "${1^^}" == "ALL" ]; then + + elif [ "${1^^}" == "SYS" ]; then [ ! -f ${PW_FOLDER}/shutdown.main ] && [ "${2}" == "TEMP" ] && SHUTDOWN_SERVERS MAIN if [ ! -f ${PW_FOLDER}/shutdown.sys ]; then touch ${PW_FOLDER}/shutdown.sys @@ -1037,47 +1044,84 @@ SHUTDOWN_SERVERS(){ fi for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Stopping iSCSI-SYS VM's on: ${ESXIHOST} ... " + echo -en "Stopping iSCSI-SYS VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 echo "Done" done for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... " + echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) echo "Done" done # sleep 1m for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Verifying iSCSI-SYS VM's stopped on: ${ESXIHOST} ... " + echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Shutting Down VM's on: ${ESXIHOST} ... " + echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " if [ "${TEST}" != "true" ]; then ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 fi echo "Done" done + for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " - [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... " + [ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background\n" + done + + fi + + elif [ "${1^^}" == "ALL" ]; then + if [ ! -f ${PW_FOLDER}/shutdown.all ]; then + touch ${PW_FOLDER}/shutdown.all + + echo -en "Disabling CLS in vCenter ... " + if [ "${TEST}" != "true" ]; then + run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + fi + echo "Done" + + for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + echo -en "Stopping all iSCSI VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " + [ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 echo "Done" done - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Shutting Down Host: ${ESXIHOST} ... " + for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) + echo "Done" + done + # sleep 1m + for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + echo -en "Verifying iSCSI-SYS VM's stopped on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo "Done" + done + for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + echo -en "Shutting Down VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " if [ "${TEST}" != "true" ]; then - ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' & - ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' + ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 + ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} & >/dev/null 2>&1 fi echo "Done" done + for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... " + [ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background\n" + done + fi - fi - - if [ "${1^^}" == "OFFSITE" ]; then + + elif [ "${1^^}" == "OFFSITE" ]; then if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then touch ${PW_FOLDER}/shutdown.offsite @@ -1089,32 +1133,35 @@ SHUTDOWN_SERVERS(){ fi echo "Done" for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... " + echo -en "Enabling Maintenance Mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) echo "Done" done for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - echo -en "Stopping VM's on: ${ESXIHOST} ... " + echo -en "Stopping VM's on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " - [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) - echo "Done" - done - for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do - echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..." - if [ "${TEST}" != "true" ]; then - run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"') - run=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0') - fi - echo "Done" + echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[${ESXIHOST}]}' ... " + [ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & + echo -e "${idsCL[Green]}Continuing in background\n" done fi fi } +MAINT_MODE_VERIFY_SHUTDOWN(){ + ESXIHOST=${1} + MaintMode="" + until [ "${MaintMode}" == "Enabled" ]; do + sleep 3s + MaintMode=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode get') + done + ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' + ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' + +} SHUTDOWN_SERVER(){ # TEST=true if [ "${PW_iDRACHOST_SHORTNAMES[${1}]}" != "" ]; then @@ -1148,28 +1195,20 @@ SHUTDOWN_SERVER(){ echo -en "${idsCL[LightCyan]}Enabling maintenance mode & starting iSCSI VM migrations ... " [ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${hostip}]} >/dev/null 2>&1 & - sleep 25s + [ "${TEST}" != "true" ] && sleep 25s echo -e "${idsCL[Green]}Done\n" - - # echo -en "${idsCL[LightCyan]}Giving time for iSCSI VM's to begin migration to other hosts ... " - # [ "${TEST}" != "true" ] && sleep 45s - # echo -e "${idsCL[Green]}Done\n" echo -en "${idsCL[LightCyan]}Shutting down remaining VM's ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${hostip} >/dev/null 2>&1) echo -e "${idsCL[Green]}Done\n" - echo -en "${idsCL[LightCyan]}Verifying no VMs remain on the host ... " - [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown-verify.ps1 ${hostip} >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done\n" + # echo -en "${idsCL[LightCyan]}Verifying no VMs remain on the host ... " + # [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown-verify.ps1 ${hostip} >/dev/null 2>&1) + # echo -e "${idsCL[Green]}Done\n" - echo -en "${idsCL[LightCyan]}Shutting down Host ... " - [ "${TEST}" != "true" ] && run=$(ssh root@${hostip} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' >/dev/null 2>&1) - echo -e "${idsCL[Green]}Host will begin shutdown in 10secs\n" - - echo -en "${idsCL[LightCyan]}Disabling maintenance mode ... " - [ "${TEST}" != "true" ] && run=$(ssh root@${hostip} 'esxcli system maintenanceMode set -e false -t 0' >/dev/null 2>&1) - echo -e "${idsCL[Green]}Done\n" + echo -en "${idsCL[LightCyan]}Waiting for VM's to power off to shutdown '${PW_ESXI_HOST_NAMES[{hostip}]}' ... " + [ "${TEST}" != "true" ] && MAINT_MODE_VERIFY_SHUTDOWN {hostip} & + echo -e "${idsCL[Green]}Continuing in background\n" if [ "${hostip}" == "10.10.2.10" ]; then echo -en "${idsCL[LightCyan]}Waiting for Host ${host} (${hostip}) to shutdown ... " @@ -1446,9 +1485,9 @@ fi logrotate) LOGROTATE;; update) - # echo -en "${idsCL[LightCyan]}Will reboot monitoring services in the background to apply updates ..." - # RESTARTALL_SERVICES >/dev/null 2>&1 & - # echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" + echo -en "${idsCL[LightCyan]}Will reboot monitoring services in the background to apply updates ..." + RESTARTALL_SERVICES >/dev/null 2>&1 & + echo -e "${idsCL[Green]}Done${idsCL[Default]}\n" ;; status) CHECKPOWER