diff --git a/defaults.inc b/defaults.inc index c066514e..5db8d24e 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='2.3.120-11022023' +VERS='2.3.130-11022023' noheader=' update service dailytemp ' @@ -121,11 +121,13 @@ PW_MAINSITE_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS}" PW_ALL_HOSTS="${PW_MAINHOSTS},${PW_SYSHOSTS},${PW_OFFSITEHOSTS}" PW_RACADM_ACCESS=10.10.0.241 +RACADMONLINE=$(CHECK_HOST ${PW_RACADM_ACCESS}) IFS=, PW_MAINSITE_HOSTS=(${PW_MAINSITE_HOSTS}) PW_MAINHOSTS=(${PW_MAINHOSTS}) PW_SYSHOSTS=(${PW_SYSHOSTS}) +PW_OFFSITEHOSTS=(${PW_OFFSITEHOSTS}) PW_ALLHOSTS=(${PW_ALL_HOSTS}) unset IFS diff --git a/powerwall.sh b/powerwall.sh index 6e0272c4..93cbe75c 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -34,7 +34,7 @@ CHECKTEMP(){ echo -e "\r\033[K${idsCL[LightCyan]}${idsST[Bold]}${SENSOR} Host Sensors ${idsST[Reset]}${idsCL[LightCyan]}- ${vHOSTiDRACIP}${idsCL[Default]}" [ "${1}" != "noavg" ] && echo -en "${idsCL[Yellow]}Pulling data and calculating averages for '${SENSOR}' ..." || echo -en "${idsCL[Yellow]}Pulling data for '${SENSOR}' ..." - if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ]; then + if [ "$(CHECK_HOST ${vHOSTIP})" != "false" ] && [ "${RACADMONLINE}" != "false" ]; then vHOSTDATA=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${vHOSTiDRACIP} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1) # iDS-vMS-Offsite Fan Issue Workaround @@ -68,7 +68,9 @@ CHECKTEMP(){ else echo -e "${idsCL[LightCyan]} ${vSENSOR}${spc}${idsCL[Default]}: No Data" fi - + elif [ "${RACADMONLINE}" == "false" ]; then + echo -e "\r\033[K${idsCL[Red]}${SENSOR} RACADM is down" + else #host down echo -e "\r\033[K${idsCL[Red]}${SENSOR} Host is down" @@ -213,7 +215,7 @@ CHECKTEMP_SERVICE(){ fi - if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then + if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ "${RACADMONLINE}" != "false" ]; then unset IFS SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1) IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS @@ -233,6 +235,9 @@ CHECKTEMP_SERVICE(){ fi fi + elif [ "${RACADMONLINE}" == "false" ]; then + SENSORS_CHECK='' + else SENSORS_CHECK=(${SENSORa}) fi @@ -451,10 +456,12 @@ DAILYTEMP(){ DIVIDER . . 60 for SENSORa in ${PW_SENSOR_ORD[@]}; do - if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ]; then + if [ "${PW_SENSOR_TYPE[${SENSORa}]}" == "esxi" ] && [ "${RACADMONLINE}" != "false" ]; then unset IFS SENSORS_CHECK=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${PW_HOST_IDRACS[${SENSORa}]} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep " Temp" 2>&1) IFS=$'\n' read -rd '' -a SENSORS_CHECK <<<"${SENSORS_CHECK}"; unset IFS + elif [ "${RACADMONLINE}" == "false" ]; then + SENSORS_CHECK='' else SENSORS_CHECK=(${SENSORa}) fi @@ -508,12 +515,14 @@ DAILYTEMP(){ GETHOSTPOWER(){ hostip=${PW_iDRACHOST_SHORTNAMES[${1}]} hostip=${PW_HOST_ADDRESSES[${hostip}]} - if [ "$(CHECK_HOST ${hostip})" != "false" ]; then + if [ "$(CHECK_HOST ${hostip})" != "false" ] && [ "${RACADMONLINE}" != "false" ]; then if [ "${1}" == "10.10.2.26" ]; then temp=0 else temp=$(ssh root@${PW_RACADM_ACCESS} racadm -r ${1} -u ${PW_ESXI_USER} -p "'${PW_ESXI_PASS}'" getsensorinfo | grep "System Board Pwr" | awk '/ / {print $6}' | sed -e 's/Watts//g' 2>&1) fi + elif [ "${RACADMONLINE}" == "false" ]; then + temp=racadmdown else temp=down fi @@ -550,8 +559,8 @@ CHECKPOWER(){ fi if [ "${1}" == "" ] || [ "${1}" == "sensors" ]; then - if [ "$(CHECK_HOST inv1.scity.us)" != "false" ]; then - INVSTATSINFO=$(curl -s "https://inv1.scity.us/stats.json") + if [ "$(CHECK_HOST 10.10.0.61)" != "false" ]; then + INVSTATSINFO=$(curl -s "http://10.10.0.61/stats.json") INPUTV=`echo ${INVSTATSINFO} | jq '.inputs .inV'` INPUTA=`echo ${INVSTATSINFO} | jq '.inputs .inA'` OUTPUTV=`echo ${INVSTATSINFO} | jq '.outputs .outV'` @@ -1006,23 +1015,23 @@ SHUTDOWN_SERVERS(){ fi echo "Done" - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do echo -en "Stopping iSCSI-PRI VM's on: ${ESXIHOST} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) echo "Done" done # sleep 1m - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do echo -en "Verifying iSCSI-PRI VM's stopped on: ${ESXIHOST} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... " if [ "${TEST}" != "true" ]; then run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) @@ -1030,7 +1039,12 @@ SHUTDOWN_SERVERS(){ fi echo "Done" done - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do + echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo "Done" + done + for ESXIHOST in "${PW_MAINHOSTS[@]}"; do echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..." if [ "${TEST}" != "true" ]; then run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"') @@ -1038,16 +1052,7 @@ SHUTDOWN_SERVERS(){ fi echo "Done" done - - # for ESXIHOST in "${PW_MAINHOSTS[@]}"; do - # echo -en "Waiting for Host (${ESXIHOST}) to shutdown... " - # if [ "${TEST}" != "true" ]; then - # while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do - # sleep 1 - # done - # fi - # echo "Done" - # done + fi fi if [ "${1^^}" == "SYS" ] || [ "${1^^}" == "ALL" ]; then @@ -1076,16 +1081,11 @@ SHUTDOWN_SERVERS(){ echo "Done" done # sleep 1m - for ESXIHOST in "${PW_MAINSITE_HOSTS[@]}"; do + for ESXIHOST in "${PW_SYSHOSTS[@]}"; do echo -en "Verifying iSCSI-SYS VM's stopped on: ${ESXIHOST} ... " [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1) echo "Done" done - # for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - # echo -en "Suspending MySQL & LAN-DNS VM's on: ${ESXIHOST} ... " - # [ "${TEST}" != "true" ] && ${PW_FOLDER}/esxi-scripts/sys-suspend.ps1 ${ESXIHOST} >/dev/null 2>&1 - # echo "Done" - # done for ESXIHOST in "${PW_SYSHOSTS[@]}"; do echo -en "Shutting Down VM's on: ${ESXIHOST} ... " if [ "${TEST}" != "true" ]; then @@ -1094,6 +1094,11 @@ SHUTDOWN_SERVERS(){ fi echo "Done" done + for ESXIHOST in "${PW_SYSHOSTS[@]}"; do + echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo "Done" + done for ESXIHOST in "${PW_SYSHOSTS[@]}"; do echo -en "Shutting Down Host: ${ESXIHOST} ... " if [ "${TEST}" != "true" ]; then @@ -1102,16 +1107,45 @@ SHUTDOWN_SERVERS(){ fi echo "Done" done - - for ESXIHOST in "${PW_SYSHOSTS[@]}"; do - echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... " + + fi + fi + + if [ "${1^^}" == "OFFSITE" ]; then + if [ ! -f ${PW_FOLDER}/shutdown.offsite ]; then + touch ${PW_FOLDER}/shutdown.offsite + + echo -en "Disabling CLS in vCenter ... " + if [ "${TEST}" != "true" ]; then + # run=$(ssh root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1) + run=$(ssh root@${PW_VCENTERHOST} "service-control --start vmware-vpxd >/dev/null & 2>&1" >/dev/null 2>&1) + fi + echo "Done" + for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do + echo -en "Enabling Maintenance Mode on: ${ESXIHOST} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXIHOST} & >/dev/null 2>&1) + echo "Done" + done + for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do + echo -en "Stopping VM's on: ${ESXIHOST} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo "Done" + done + for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do + echo -en "Waiting for all VM's to be stopped on: ${ESXIHOST} ... " + [ "${TEST}" != "true" ] && run=$(${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1) + echo "Done" + done + for ESXIHOST in "${PW_OFFSITEHOSTS[@]}"; do + echo -en "Shutting down host & disabling Maintenance Mode: ${ESXIHOST} ..." if [ "${TEST}" != "true" ]; then - while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do - sleep 1 - done + run=$(ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"') + run=$(ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0') fi echo "Done" done + fi fi }