#!/usr/bin/env bash # powerwall - CLI commands to control VM guest power PW_ACTION="${1}" source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/powerwall/defaults.inc source /opt/idssys/powerwall/inc/power.inc.sh source /opt/idssys/powerwall/inc/temp.inc.sh shopt -s lastpipe VERBOSE=false RESTARTALL_SERVICES(){ echo STOPALL_SERVICES echo -en "${idsCL[Yellow]}Waiting${idsCL[Default]}" sleep 6s echo -en "\033[2K\r" DIVIDER . yellow 50 STARTALL_SERVICES echo } STOPALL_SERVICES(){ for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do SERVICE_MNG ${srvc} stop # echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[${srvc}]}' Service Stopped" >> ${PW_LOGFILE} done } STARTALL_SERVICES(){ for srvc in "${!PW_POWERWALL_SERVICES[@]}"; do SERVICE_MNG ${srvc} start # echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[${srvc}]}' Service Started" >> ${PW_LOGFILE} done } SERVICE_MNG(){ if [ "${2}" == "start" ]; then sa="${idsCL[Green]}Starting" elif [ "${2}" == "stop" ]; then sa="${idsCL[LightRed]}Stopping" elif [ "${2}" == "restart" ]; then sa="${idsCL[Green]}Restarting" elif [ "${2}" == "status" ]; then sa="${idsCL[Yellow]}Status for" fi MSG="${sa} Service - ${idsCL[LightCyan]}${PW_POWERWALL_SERVICES[${1}]}" c=0; spc=''; spc1=`expr 53 - ${#MSG}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done echo -en "${idsCL[LightCyan]}${MSG}${spc}${idsCL[Default]}: " /bin/systemctl ${2} ${PW_POWERWALL_SERVICES[${1}]} echo -e "${idsCL[Green]}Done${idsCL[Default]}" } STOP_SERVICE(){ ps -ef | grep "powerwall.*service ${1} start" | grep -v "grep" | awk '{print $2}' | xargs kill } CHECK_SERVICES(){ echo echo -e "${idsCL[Yellow]}PowerWall Monitor Service Status" DIVIDER . yellow ${PRI_WIDTH} for srvc in "${PW_POWERWALL_SERVICES[@]}"; do c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done echo -en "${idsCL[LightCyan]}${srvc}${spc}: " [ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running" echo -e "${idsCL[Default]}" done echo } SHUTDOWN_MAIN(){ if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then touch ${PW_TMPFOLDER}/shutdown.MAIN.started SHUTDOWN_SERVER main -esxi off [ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!' sendnotice=true else sendnotice=false fi if [ "${sendnotice}" == "true" ]; then SENDNOTICE "MAIN SERVERS SHUTDOWN" "MAIN servers have been shutdown ${reason}" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - Main servers have been shutdown, ${reason}" >> ${PW_LOGFILE} fi echo } SHUTDOWN_SYS(){ if [ "${1}" == "ServerRoomTH" ] || [[ "${1}" = *"ServerRoomLA"* ]] || [ "${1}" == "TEMP" ] || [ "${1}" == "POWER" ]; then if [ ! -f ${PW_TMPFOLDER}/shutdown.MAIN ]; then touch ${PW_TMPFOLDER}/shutdown.MAINSITE.started SHUTDOWN_SERVER mainsite -esxi off else touch ${PW_TMPFOLDER}/shutdown.SYS.started SHUTDOWN_SERVER sys -esxi off fi [ "${1}" == "POWER" ] && reason='Low Power!' || reason='Server Room Overheated!' sendnotice=true else sendnotice=false fi if [ "${sendnotice}" == "true" ]; then SENDNOTICE "SYS/MAINSITE SERVERS SHUTDOWN" "SYS servers have been shutdown ${reason}" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - SYS servers have been shutdown, ${reason}" >> ${PW_LOGFILE} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} fi # curl -m 3 --data "cmd=PWRoff" http://${PW_INVERTER_IP}/cmd echo } SHUTDOWN_CRIT(){ if [ "${1}" == "Powerwall-CPU" ]; then SENDNOTICE "${1} TEMP CRITICAL" "Powerwall System shutting down" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - Powerwall System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - Powerwall System shutting down" >> ${PW_LOGFILE} sleep 1s /sbin/poweroff elif [ "${1}" == "OctoPi-CPU" ]; then SENDNOTICE "${1} TEMP CRITICAL" "OctoPi System shutting down" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL TEMP - OctoPi System shutting down" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} TEMP CRITICAL - OctoPi System shutting down" >> ${PW_LOGFILE} sleep 1s ${SSHCMD} root@${PW_REMOTE_SENSORS[${1}]} /sbin/poweroff elif [ "${1}" == "OctoPi-CPU-Usage" ]; then SENDNOTICE "${1} USAGE CRITICAL" "OctoPi System rebooting" 1 echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - CRITICAL USAGE - OctoPi System rebooting" >> ${PW_LOGFOLDER}/log-temp-${1} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - ${1} USAGE CRITICAL - OctoPi System rebooting" >> ${PW_LOGFILE} ${SSHCMD} root@${PW_REMOTE_SENSORS[${1}]} /sbin/reboot fi } SHUTDOWN_OFFSITE(){ if [ ! -f ${PW_TMPFOLDER}/shutdown.OFFSITE ]; then SENDNOTICE "OFFSITE SHUTDOWN - POWER OFF/LOW!!" "Shutting down offsite host(s)" 1 for offsite_host in "${PW_OFFSITEHOSTS[@]}"; do SHUTDOWN_SERVER ${offsite_host} -esxi off & done fi } CONSERVE_POWER(){ for conserve_host in "${PW_CONSERVEHOSTS[@]}"; do SHUTDOWN_SERVER ${conserve_host} -esxi off & done } COOLDOWN_SRVRM(){ ${PW_FOLDER}/esxi-scripts/cooldown.ps1 sleep 20 SHUTDOWN_SERVER 10.10.2.16 -esxi off } MAINT_MODE_VERIFY_SHUTDOWN(){ ESXIHOST=${1} MaintMode="" until [ "${MaintMode}" == "Enabled" ]; do sleep 3s MaintMode=$(${SSHCMD} root@${ESXIHOST} 'esxcli system maintenanceMode get') done rm -f ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.* touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.disable ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXIHOST}]}.shutdown ${SSHCMD} root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Power-Monitor script initiated system shutdown"' sleep 8.75s ${SSHCMD} root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0' } SHUTDOWN_SERVER(){ if [ "${1}" != "-h" ]; then HOST=${1}; MN=0 if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]} ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]} elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then ESXI_HOST=${HOST} elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]} elif ([ "${HOST::1}" == "o" ] || [ "${HOST::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]} elif ([ "${HOST::1}" == "k" ] || [ "${HOST::1}" == "K" ]) && [ "${PW_HOST_ADDRESSES[KP-vMS-Host${HOST: -1}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[KP-vMS-Host${HOST: -1}]} elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]} elif [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ] || [ "${HOST^^}" == "CONSERVE" ]; then ([ "${HOST^^}" == "MAIN" ] && [ -f ${PW_TMPFOLDER}/power.ac.conserve ]) && ESXI_HOST='MAIN_NOISCSI' || ESXI_HOST=${HOST^^} MN=1 else ESXI_HOST='' fi [ "${ESXI_HOST}" == "" ] && echo -e "Invalid host\n" && exit 1 if [ "${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}" != "" ]; then tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} idracip=${PW_HOST_IDRACS[${tmp}]} hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]} fi fi while [ $# -gt 0 ]; do case "$1" in -p|-power) POWER=${2}; MODE="power";; -e|-esxi) ESXI=${2}; MODE="esxi";; -h) echo -e "Usage: ${idsCL[Yellow]}pw host {host} {flags}${idsCL[Default]} {" width=35 printf "%-${width}s- %s\n" " {host}" "(*required, can either be esxiIP,idracIP or Host#)" printf "%-${width}s- %s\n" " -p,-power {on / off}" "Controls Host Power" printf "%-${width}s- %s\n" " -e,-esxi {off / mm}" "Shuts down ESXi or puts into maintenance mode" echo -e "}\n" exit 0 ;; esac shift 1 done ([ "${ESXI,,}" == "maint" ] || [ "${ESXI,,}" == "maintenance" ] || [ "${ESXI,,}" == "mm" ]) && MM=1 || MM=0 if [ ${MN} -eq 1 ]; then ESXI_HOSTS=PW_${ESXI_HOST}HOSTS[@] ESXI_HOSTS=(${!ESXI_HOSTS}) [ "${ESXI_HOST}" == "ALL" ] && hostname="All Host Servers" || hostname="All ${ESXI_HOST} Host Servers" else ESXI_HOSTS=(${ESXI_HOST}) hostname=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} fi if [[ "${IP}" == "10.6.1"* ]]; then for ESXIHOST in "${ESXI_HOSTS[@]}"; do MSG="Shutting down: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} if [ ${TESTMODE} -eq 0 ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then ${SSHCMD} root@${ESXIHOST} "/vmfs/volumes/KP-Datastore1 (R6) (0-13)/esxidown/async.sh" echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done else echo DIVIDER . yellow ${PRI_WIDTH} [ ${MM} -eq 1 ] && smsg="Entering Maintenance Mode" || smsg="Shutting down" echo -e "${idsCL[LightYellow]}${hostname}${idsCL[Yellow]} - ${smsg}${idsCL[Default]}" DIVIDER . yellow ${PRI_WIDTH} echo if [ "${ESXI_HOSTS}" != "" ]; then if [ ${MM} -eq 1 ]; then echo "($(date +'%Y-%m-%d %H:%M:%S')) - Putting ${hostname} into maintenance mode" >> ${PW_LOGFILE} else echo "($(date +'%Y-%m-%d %H:%M:%S')) - Shutting down ${hostname}" >> ${PW_LOGFILE} fi ##INCREASE CPU OVERCOMMIT if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${ESXI_HOST}" != "OFFSITE" ]; then MSG="Increasing DRS CPU-Overcommit to 4" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then ${PW_FOLDER}/esxi-scripts/set-drs.ps1 4 >/dev/null 2>&1 & echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" fi fi if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then MSG="Stopping all iSCSI-PRI VM's" elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then MSG="Stopping all iSCSI-SYS VM's" elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then MSG="Stopping all iSCSI VM's" fi c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 & elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 & elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 >/dev/null 2>&1 & fi sleep 5s echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi echo fi if [ ${MM} -eq 0 ] && [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then # if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "OFFSITE" ]; then # MSG="Disabling CLS for the Offsite Cluster in vCenter" # c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done # echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " # if [ ${TESTMODE} -eq 0 ]; then # run=$(${SSHCMD} root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) # echo -e "${idsCL[Green]}Done${idsCL[Default]}" # touch ${PW_TMPFOLDER}/shutdown.cls.offsite # else # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" # fi # fi if [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ]); then MSG="Disabling CLS for the Primary Cluster in vCenter" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then run=$(${SSHCMD} root@${PW_VCENTERHOST} "sed -i '//{n;s/.*<\/enabled>/False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1) echo -e "${idsCL[Green]}Done${idsCL[Default]}" touch ${PW_TMPFOLDER}/shutdown.cls.mainsite else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi fi MSG="Restarting vCenter CLS Services" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then ${SSHCMD} root@${PW_VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1 && service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null & 2>&1 & echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi echo fi for ESXIHOST in "${ESXI_HOSTS[@]}"; do MSG="Enabling maintenance mode on: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} if [ ${TESTMODE} -eq 0 ]; then if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then ${PW_FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${PW_VCENTER_HOST_NAMES[${ESXIHOST}]} >/dev/null 2>&1 & sleep 10s echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done echo if [ ${MM} -eq 0 ] && ([[ "${ESXI_HOST}" == *"10.6.1."* ]] || [ "${ESXI_HOST}" == "ALL" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || [ "${ESXI_HOST}" == "OFFSITE" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [[ "${PW_OFFSITE_HOSTS}" = *"${ESXI_HOST}"* ]] || ([ "${ESXI_HOST}" == "SYS" ] && [ -f ${PW_TMPFOLDER}/shutdown.MAIN ])); then tmp=tmp else MSG="Starting iSCSI migrations" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then sleep 80s # up from 45s, then 60... echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi echo fi if [[ "${ESXI_HOST}" != *"10.6.1."* ]] && ([ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "SYS" ] || [ "${ESXI_HOST}" == "CONSERVE" ]); then MSG="Waiting for iSCSI VM's to power down" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${idsCL[White]}${MSG_SPC}: " # for ESXIHOST in "${PW_MAINSITEHOSTS[@]}"; do if [ ${TESTMODE} -eq 0 ]; then if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1 elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1 elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1 fi echo -e "${idsCL[Green]}Done${idsCL[Default]}" # if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${PW_ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ] && [ "$(${SSHCMD} root@${ESXIHOST} 'esxcli system maintenanceMode get')" != "Enabled" ]; then # VERIFY_ISCSI_VM_DOWN "${ESXI_HOST}" & # else # touch ${PW_TMPFOLDER}/verify.iscsi.${ESXIHOST}.done # fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi # done # if [ ${TESTMODE} -eq 0 ]; then # nc_count=0 # until [ -f ${PW_TMPFOLDER}/verify.iscsi.done ]; do # sleep 1s # done # echo -e "${idsCL[Green]}Done${idsCL[Default]}" # else # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" # fi if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then MSG="Shutting down iSCSI-PRI Server" elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then MSG="Shutting down iSCSI-SYS Server" elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then MSG="Shutting down both iSCSI-PRI & iSCSI-SYS Servers" fi c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then if [ "${ESXI_HOST}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & elif [ "${ESXI_HOST}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${ESXI_HOST}" == "SYS" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & elif [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_PRI_HOST} >/dev/null 2>&1 & ${PW_FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${PW_ISCSI_SYS_HOST} >/dev/null 2>&1 & fi echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi echo fi MSG="Shutting down any remaining VM's" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then for ESXIHOST in "${ESXI_HOSTS[@]}"; do varvms="${PW_VCENTER_HOST_NAMES[${ESXIHOST}]},${varvms}" done ${PW_FOLDER}/esxi-scripts/vm-shutdown.ps1 "${varvms::-1}" >/dev/null 2>&1 & echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi echo if [ ${MM} -eq 0 ]; then for ESXIHOST in "${ESXI_HOSTS[@]}"; do MSG="Waiting for VM's, will then shutdown: ${PW_ESXI_HOST_NAMES[${ESXIHOST}]}" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_ESXI_HOST_NAMES[${ESXIHOST}]}/${idsCL[LightCyan]}${PW_ESXI_HOST_NAMES[${ESXIHOST}]}}${idsCL[White]}${MSG_SPC}: " [[ "${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]}" = "10.6."* ]] && ESXI_PASS=${PW_ESXI_PASS_KP} || ESXI_PASS=${PW_ESXI_PASS} if [ ${TESTMODE} -eq 0 ] && [ "${TEST_DONOT_SHUTDOWN^^}" != "TRUE" ]; then if [ "$(ipmitool -I lanplus -H ${PW_HOST_IDRACS[${PW_ESXI_HOST_NAMES[${ESXIHOST}]}]} -U ${PW_ESXI_USER} -P "${ESXI_PASS}" chassis power status | grep 'Power is on')" != "" ]; then MAINT_MODE_VERIFY_SHUTDOWN ${ESXIHOST} & echo -e "${idsCL[Green]}Continuing in background${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Host Offline${idsCL[Default]}" fi else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi done echo if [ "${ESXI_HOST}" == "${PW_MAIN_ISCSI}" ] || [ "${ESXI_HOST}" == "MAIN" ] || [ "${ESXI_HOST}" == "MAINSITE" ] || [ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "CONSERVE" ]; then MSG="Waiting for ${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]} to shutdown" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG//${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}/${idsCL[LightCyan]}${PW_VCENTER_HOST_NAMES[${PW_MAIN_ISCSI}]}${idsCL[White]}}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then while ping -qw 10 -c3 "${PW_MAIN_ISCSI}">/dev/null 2>&1; do sleep 1; done sleep 5s echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi MSG="Powering off SAS Enclosure" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ] && [ "$(CHECK_HOST ${PW_MAIN_ISCSI})" == "false" ]; then run=$(snmpset -v 1 -c private ${PW_APCPDU_IP} .1.3.6.1.4.1.318.1.1.12.3.3.1.1.4.14 i 2) echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}" fi fi [ "${ESXI_HOST}" != "${PW_MAIN_ISCSI}" ] && echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be shutdown momentarily ... \n" touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.shutdown touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.disable else echo -e "\n${idsCL[LightCyan]}${hostname}${idsCL[White]} will be in maintenance mode momentarily ... \n" touch ${PW_TMPFOLDER}/${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}.maintmode fi ## DECREASE CPU OVERCOMMIT if [[ "${PW_OFFSITE_HOSTS}" != *"${ESXI_HOST}"* ]] && [ "${HOST^^}" != "OFFSITE" ]; then MSG="Decreasing DRS CPU-Overcommit back down to 2" c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " if [ ${TESTMODE} -eq 0 ]; then { sleep 60s; ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1; } & echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" else echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" fi fi else echo -e "${idsCL[LightRed]}Host cannot be found for '${1}'${idsCL[Default]}" fi fi echo } VERIFY_ISCSI_VM_DOWN(){ if [ "${1}" == "${PW_ISCSI_PRI_HOST}" ] || [ "${1}" == "MAIN" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 >/dev/null 2>&1 elif [ "${1}" == "${PW_ISCSI_SYS_HOST}" ] || [ "${1}" == "SYS" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown-verify.ps1 >/dev/null 2>&1 elif [ "${1}" == "MAINSITE" ] || [ "${1}" == "ALL" ]; then ${PW_FOLDER}/esxi-scripts/iscsi-all-shutdown-verify.ps1 >/dev/null 2>&1 fi touch ${PW_TMPFOLDER}/verify.iscsi.done } HOSTMGMT(){ echo -e "\n${idsCL[Yellow]}Host Controls${idsCL[Default]}" DIVIDER . yellow ${PRI_WIDTH} HOST=${1} if [ "${HOST}" != "-s" ] && [ "${HOST,,}" != "-status" ] && [ "${HOST}" != "-h" ] && [ "${HOST,,}" != "-help" ] && [ "${HOST}" != "" ]; then cHOST=${HOST} MN=0 if [ "${PW_iDRACHOST_SHORTNAMES[${HOST}]}" != "" ]; then tmp=${PW_iDRACHOST_SHORTNAMES[${HOST}]} ESXI_HOST=${PW_HOST_ADDRESSES[${tmp}]} elif [ "${PW_ESXI_HOST_NAMES[${HOST}]}" != "" ]; then ESXI_HOST=${HOST} elif [ "${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[iDS-vMS-Host${HOST}]} elif ([ "${HOST::1}" == "o" ] || [ "${HOST::1}" == "O" ]) && [ "${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[OFF-vMS-Host${HOST: -1}]} elif ([ "${HOST::1}" == "k" ] || [ "${HOST::1}" == "K" ]) && [ "${PW_HOST_ADDRESSES[KP-vMS-Host${HOST: -1}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[KP-vMS-Host${HOST: -1}]} elif [ "${PW_HOST_ADDRESSES[${HOST}]}" != "" ]; then ESXI_HOST=${PW_HOST_ADDRESSES[${HOST}]} elif [ "${HOST^^}" == "ALL" ] || [ "${HOST^^}" == "MAINSITE" ] || [ "${HOST^^}" == "OFFSITE" ] || [ "${HOST^^}" == "MAIN" ] || [ "${HOST^^}" == "SYS" ] || [ "${HOST^^}" == "CONSERVE" ]; then ESXI_HOST=${HOST^^} MN=1 else ESXI_HOST='' fi if [ "${ESXI_HOST}" == "" ]; then echo -e "Invalid host\n" exit 1 fi if [ ${MN} -eq 1 ]; then [ "${ESXI_HOST}" == "ALL" ] && hostname="All Host Servers" || hostname="All ${ESXI_HOST} Host Servers" else tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} idracip=${PW_HOST_IDRACS[${tmp}]} hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]} fi if [ "${PW_ESXI_HOST_NAMES[${ESXI_HOST}]}" != "" ]; then tmp=${PW_ESXI_HOST_NAMES[${ESXI_HOST}]} idracip=${PW_HOST_IDRACS[${tmp}]} hostname=${PW_iDRACHOST_SHORTNAMES[${idracip}]} fi elif [ "${HOST}" == "-s" ] || [ "${HOST,,}" == "-status" ]; then for ESXIHOST in "${PW_ALLHOSTS[@]}"; do echo ${ESXIHOST} done exit 0 else echo -e "Usage: ${idsCL[Yellow]}pw host {host} {flags}${idsCL[Default]} {" width=35 printf "%-${width}s- %s\n" " {host}" "(*required if not uing -s, can either be esxiIP,idracIP or Host#)" printf "%-${width}s- %s\n" " -p,-power {on / off}" "Controls Host Power" printf "%-${width}s- %s\n" " -e,-esxi {off / mm}" "Shuts down ESXi or puts into maintenance mode" printf "%-${width}s- %s\n" " -s,-status" "List all Host's Status" echo -e "}\n" exit 0 fi while [ $# -gt 0 ]; do case "$1" in -p|-power) POWER=${2}; MODE="power";; -e|-esxi) ESXI=${2}; MODE="esxi";; -y) FORCE=true;; esac shift 1 done [ ${MN} -eq 1 ] && ssppcc=31 || ssppcc=21 ([ "${ESXI,,}" == "maint" ] || [ "${ESXI,,}" == "maintenance" ] || [ "${ESXI,,}" == "mm" ]) && MM=1 || MM=0 c=0; spcA=''; spc1=`expr ${ssppcc} - ${#hostname} - 1`; until [ ${c} = ${spc1} ]; do spcA="${spcA} "; ((c++)); done echo -en "${idsCL[LightCyan]} Host: ${idsCL[Green]}${hostname}${spcA}" if [ ${MN} -eq 1 ]; then echo -e "" else c=0; spcB=''; spc1=`expr ${ssppcc} - 3 - ${#ESXI_HOST} - 1`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; ((c++)); done echo -en "${idsCL[LightCyan]}Host IP: ${idsCL[Green]}${ESXI_HOST}${spcB}" echo -e "${idsCL[LightCyan]}iDrac IP: ${idsCL[Green]}${idracip}${idsCL[Default]}" fi DIVIDER . yellow ${PRI_WIDTH} if [ ${MN} -eq 1 ]; then var=PW_${ESXI_HOST}HOSTS[@]; v=1 for vhost in "${!var}"; do echo -e " ${v})${idsCL[LightCyan]} ${PW_ESXI_HOST_NAMES[${vhost}]} [${vhost}]${idsCL[Default]}" ((v++)) done DIVIDER . yellow ${PRI_WIDTH} fi echo ([ "${HOST}" == "" ] || ([ "${POWER}" == "" ] && [ "${ESXI}" == "" ])) && echo -e "Invalid command arguments\n" && exit 1 if [ ${TESTMODE} -eq 0 ] && [ "${FORCE}" != "true" ] && ([ "${POWER^^}" == "OFF" ] || [ "${ESXI^^}" == "OFF" ] || [ ${MM} -eq 1 ]); then if [ ${MM} -eq 1 ]; then read -n 1 -p "Are you sure you wish to put '${ESXI_HOST}' into maintenance mode ? (y/N): " choice else read -n 1 -p "Are you sure you wish to power down '${ESXI_HOST}' ? (y/N): " choice fi [ "${choice^^}" != "Y" ] && exit 0 echo -e "\n" fi if [ "${MODE}" == "esxi" ]; then if [ ${MN} -eq 1 ]; then if [ ${MM} -eq 1 ]; then echo -e "${idsCL[LightGreen]}PUTTING '${ESXI_HOST}' HOSTS INTO MAINTENANCE MODE${idsCL[Default]}" SHUTDOWN_SERVER ${ESXI_HOST,,} -esxi maintenance else echo -e "${idsCL[LightGreen]}SHUTTING DOWN '${ESXI_HOST}' HOSTS${idsCL[Default]}" SHUTDOWN_SERVER ${ESXI_HOST,,} -esxi off fi else if [ ${MM} -eq 1 ]; then echo -e "${idsCL[LightGreen]}PUTTING '${ESXI_HOST}' HOST INTO MAINTENANCE MODE${idsCL[Default]}" SHUTDOWN_SERVER ${ESXI_HOST} -esxi maintenance else echo -e "${idsCL[LightGreen]}SHUTTING DOWN HOST${idsCL[Default]}" SHUTDOWN_SERVER ${ESXI_HOST} -esxi off fi fi elif [ "${MODE}" == "power" ]; then if [ ${MN} -eq 1 ]; then var=PW_${ESXI_HOST}HOSTS[@]; v=1 for vhost in "${!var}"; do HOSTPOWER ${vhost} ${POWER} sleep 3s done # if [ "${POWER^^}" == "ON" ] && ([ "${ESXI_HOST}" == "ALL" ] || [ "${ESXI_HOST}" == "MAINSITE" ]); then # ##DECREASE CPU OVERCOMMIT # MSG="Decreasing DRS CPU-Overcommit back down to 2" # c=0; MSG_SPC=''; spc1=`expr ${HC_CW} - ${#MSG}`; until [ ${c} = ${spc1} ]; do MSG_SPC="${MSG_SPC} "; ((c++)); done # echo -en "${idsCL[White]}${MSG}${MSG_SPC}: " # if [ ${TESTMODE} -eq 0 ]; then # ${PW_FOLDER}/esxi-scripts/set-drs.ps1 2 >/dev/null 2>&1 & # echo -e "${idsCL[Green]}Command Sent${idsCL[Default]}\n" # else # echo -e "${idsCL[Yellow]}(TEST MODE)${idsCL[Default]}\n" # fi # fi else HOSTPOWER ${ESXI_HOST} ${POWER} fi fi echo } ISCSIVMSHUTDOWN(){ echo -en "${idsCL[LightCyan]}Shutting down any VM's that are running on an iSCSI-PRI drive ... " ${PW_FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 >/dev/null 2>&1 & echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" echo echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..." echo } ISCSISYSSHUTDOWN(){ echo -en "${idsCL[LightCyan]}Shutting down any VM's that are running on an iSCSI-SYS drive ... " ${PW_FOLDER}/esxi-scripts/iscsi-sys-vm-shutdown.ps1 >/dev/null 2>&1 & echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" echo echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..." echo } ISCSIALLSHUTDOWN(){ echo -en "${idsCL[LightCyan]}Shutting down any VM's that are running on any iSCSI drive ... " ${PW_FOLDER}/esxi-scripts/iscsi-all-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1 & echo -e "${idsCL[Green]}Command sent${idsCL[Default]}" echo echo -e "${idsCL[LightCyan]}Please wait a few minutes while all iSCSI guests are shutdown ..." echo } LOGROTATE(){ if [ -f ${NM_TMPFOLDER}/.singleserver.PW.primary ] || [ -f ${NM_TMPFOLDER}/.singleserver.PWO.primary ]; then STOPALL_SERVICES lm=`expr $(date +'%m') - 1` if [ ${lm} = 0 ]; then lm=12 ly=`expr $(date +'%Y') - 1` else ly=$(date +'%Y') fi [ ${lm} -lt 10 ] && lm="0${lm}" ARCHIVEFOLDER=${PW_LOGARCHIVE}/${ly}-${lm} mkdir -p ${ARCHIVEFOLDER} sleep 5s mv ${PW_LOGFOLDER}/log* ${ARCHIVEFOLDER}/ 2> /dev/null mv ${PW_LOGFOLDER}/*.log ${ARCHIVEFOLDER}/ 2> /dev/null mv ${PW_LOGFOLDER}/service.* ${ARCHIVEFOLDER}/ 2> /dev/null # source ${PW_FOLDER}/defaults.inc STARTALL_SERVICES backupstokeep=(`ls -d ${PW_LOGARCHIVE}/* 2> /dev/null | sort | tail -12`) for folder in $(ls -d ${PW_LOGARCHIVE}/* | sort -r); do keep=0 for filetokeep in ${backupstokeep[@]}; do if [ "${folder}" == "${filetokeep}" ]; then keep=1; break fi done; [ ${keep} == 0 ] && rm -Rf ${folder} done fi } GET_SNAPSHOTS(){ echo -e "\n${idsCL[Yellow]}LISTING ALL VM SNAPHOTS IN VCENTER ${idsCL[Default]}(Excluding Replicas)" DIVIDER . lightGreen ${PRI_WIDTH} ${PW_FOLDER}/esxi-scripts/get-snapshots.ps1 echo } CONFSYNC(){ [ "${PW_OFFSITE_MONITOR}" != "" ] && rsync -az --timeout=3 ${PW_FOLDER}/conf root@${PW_OFFSITE_MONITOR}:${PW_FOLDER}/ & >/dev/null 2>&1 if [ -f ${NM_FOLDER}/conf/defaults.local.inc ]; then source ${NM_FOLDER}/defaults.inc rsync -az --timeout=3 ${PW_FOLDER}/conf/* root@${NM_NODEMANAGER}:${NM_FOLDER}/conf/powerwall/ & >/dev/null 2>&1 # rsync -az --timeout=3 ${PW_FOLDER}/conf/settings.ps1 root@${NM_NODEMANAGER}:${NM_FOLDER}/conf/powerwall.settings.ps1 & >/dev/null 2>&1 fi } UPDATE_TINYTUYA_SERVER(){ if [ -f ${NM_TMPFOLDER}/.singleserver.PW.primary ] || [ -f ${NM_TMPFOLDER}/.singleserver.PWO.primary ]; then TTS_FLDR=/opt/tinytuya-server if [ ! -f /opt/raspiBackup.running ] && ([ -f ${NM_TMPFOLDER}/.singleserver.PW.primary ] || [ -f ${NM_TMPFOLDER}/.singleserver.PWO.primary ]); then if [ -d ${TTS_FLDR}/devices.json ]; then docker stop tinytuya-server rm -Rf ${TTS_FLDR}/devices.json fi python3 -m tinytuya wizard -y -no-poll -credentials-file ${TTS_FLDR}/tinytuya.json -raw-response-file ${TTS_FLDR}/tuya-raw.json -device-file ${TTS_FLDR}/devices.json sleep 1s docker restart tinytuya-server fi fi } DOWNLIST(){ DOWN_LIST=`find ${PW_TMPFOLDER}/ -maxdepth 1 -iname "*.down"` IFS=$'\n' read -rd '' -a DOWN_LIST <<<"${DOWN_LIST}" unset IFS if [ "${1}" == "report" ]; then unset idsCL idsBG idsST idsCL=('') idsBG=('') idsST=('') DOWNLIST=() for dli in "${DOWN_LIST[@]}"; do [ ! -f ${dli//.down/.disable} ] && DOWNLIST+=(${dli}) done else DOWNLIST=(${DOWN_LIST[@]}) fi echo if [ "${DOWNLIST}" != "" ]; then [ "${1}" == "report" ] && dlen=100 || dlen=110 echo -e "${idsCL[LightGreen]}Current List of Sensors/Devices Down...${idsCL[Default]}" DIVIDER false yellow ${dlen} if [ "${1}" == "report" ]; then echo -e "${idsCL[Yellow]}[Type] [Sensor_Name] [Data_From] [Down_Time]${idsCL[Default]}" else echo -e "${idsCL[Yellow]} [Type] [Sensor_Name] [Data_From] [Down_Time] [Notifications]${idsCL[Default]}" fi DIVIDER false yellow ${dlen} until [ "${choice^^}" = "E" ]; do cw=26; cw1=12; cw2=20; is=1; for downitem in ${DOWNLIST[@]}; do ditem=${downitem##*/}; ditem=${ditem//.down} if [ "${PW_SENSOR_TYPE[${ditem}]}" == "esxi" ]; then itemname=${ditem} itemtype=ESXi-HOST itemaddr=${PW_HOST_ADDRESSES[${ditem}]} elif [ "${PW_SENSOR_TYPE[${ditem}]}" == "system" ] || [ "${PW_SENSOR_TYPE[${ditem}]}" == "cpu" ] || [ "${PW_SENSOR_TYPE[${ditem}]}" == "remotesystem" ]; then itemname=${ditem} itemtype=DEVICE itemaddr=${PW_REMOTE_SENSORS[${ditem}]} elif [ "${PW_SENSOR_TYPE[${ditem}]}" == "mqtt" ]; then itemname=${ditem} itemtype=MQTT itemaddr=MQTT-Server else itemname=${ditem} itemtype=SENSOR itemaddr='' fi if [ -f ${PW_TMPFOLDER}/${ditem}-error.reading ]; then toterrtime=$(expr $(date +%s) - $(stat -c %Y ${PW_TMPFOLDER}/${ditem}-error.reading)) else toterrtime=$(expr $(date +%s) - $(stat -c %Y ${PW_TMPFOLDER}/${ditem}.down)) fi c=0; spc=''; spc1=`expr ${cw} - ${#itemname}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done c=0; spca=''; spc1=`expr ${cw1} - ${#itemtype}`; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done c=0; spc2=''; spc1=`expr ${cw2} - ${#itemaddr}`; until [ $c = ${spc1} ]; do spc2="${spc2} "; c=`expr $c + 1`; done time=$(SHOW_TIME ${toterrtime}) if [ "${1}" == "report" ]; then echo -e "${idsCL[LightRed]}${itemtype}${spca}${itemname}${spc}${itemaddr}${spc2}${time}${idsCL[Default]}" else if [ "${choice^^}" == "A" ] || ([[ ${choice} == ?(-)+([0-9]) ]] && [ ${choice} -gt 0 ] 2>/dev/null && [ ${is} -eq ${choice} ]); then [ ! -f ${PW_TMPFOLDER}/${ditem}.disable ] && touch ${PW_TMPFOLDER}/${ditem}.disable || rm -f ${PW_TMPFOLDER}/${ditem}.disable fi [ ${is} -lt 10 ] && ignsel=" ${is}) " || ignsel="${is}) " if [ -f ${PW_TMPFOLDER}/${ditem}.shutdown ]; then dstatus="SHUTDOWN" elif [ -f ${PW_TMPFOLDER}/${ditem}.disable ]; then dstatus="OFF" else dstatus="${idsCL[Green]}ON" fi c=0; spc3=''; until [ $c = `expr 26 - ${#time}` ]; do spc3="${spc3} "; ((c++)); done echo -e "\033[K${idsCL[LightRed]}${idsST[Bold]}${ignsel}${idsST[Reset]}${idsCL[LightRed]}${itemtype}${spca}${itemname}${spc}${itemaddr}${spc2}${time}${spc3}${dstatus}${idsCL[Default]}" fi DIVIDER false darkGray ${dlen} ((is++)) done if [ "${1}" != "report" ]; then echo -en "\n\033[K${idsCL[LightCyan]}Enter # to change notifications [${idsCL[Default]}Change (${idsCL[LightYellow]}A${idsCL[Default]})ll, ${idsCL[Default]}(${idsCL[LightYellow]}E${idsCL[Default]})xit${idsCL[LightCyan]}] : ${idsCL[Default]}" read choice else choice=E fi [ "${choice^^}" == "E" ] && break echo -e "\033[$(echo "scale=1; (${#DOWNLIST[@]}*2)+3" | bc)A" for (( c=1; c<$(echo "scale=1; (${#DOWNLIST[@]}*2)+3" | bc); c++ )); do echo -e "\033[K"; done echo -e "\033[$(echo "scale=1; (${#DOWNLIST[@]}*2)+3" | bc)A" done echo fi echo } ######################################################### if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [[ "${noheader}" != *" ${3} "* ]] && [[ "${noheader}" != *" ${4} "* ]] && [[ "${noheader}" != *" ${5} "* ]]; then if [ "${1}" != "check" ] || ([ "${1}" == "check" ] && [ "${2}" != "all" ]); then echo DIVIDER . lightGreen ${PRI_WIDTH} msg="|| PowerWall Management ${idsCL[Default]}" c=0; spc=''; spc1=`expr 77 - ${#msg} - ${#PW_VERS}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done echo -e "${idsCL[LightGreen]}${msg}${spc}${idsCL[LightGray]}v${PW_VERS} ${idsCL[LightGreen]}||${idsCL[Default]}" DIVIDER . lightGreen ${PRI_WIDTH} fi fi # if [ ${PW_ACTION-x} ]; then case ${PW_ACTION} in services) [ "${2}" == "stop" ] && STOPALL_SERVICES [ "${2}" == "start" ] && STARTALL_SERVICES [ "${2}" == "restart" ] && RESTARTALL_SERVICES [ "${2}" == "status" ] && CHECK_SERVICES ;; dailytemp) start=`date +%s` DAILYTEMP # SENDNOTICE "Daily Temp Readings" "$(DAILYTEMP)" end=`date +%s`; runtime=$((end-start)) echo -e "\nscript runtime: ${runtime}s\n" ;; checktemp) CHECKTEMP ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};; checkpower) CHECKPOWER ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};; check) ([ "${2}" == "" ] || ([ "${2}" != "power" ] && [ "${2}" != "temp" ] && [ "${2}" != "all" ])) && checkrun="-help" || checkrun=${2} case ${checkrun} in power) CHECKPOWER ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};; temp) CHECKTEMP ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10};; all) ${PW_SCRIPT} check temp ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${PW_SCRIPT} check power ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ;; -h|-help) echo -e "\nUsage: ${idsCL[Yellow]}[powerwall or pw] ${idsCL[LightYellow]}check ${idsCL[LightCyan]}{type} {flags} ${idsCL[Default]} {\n" width=35 printf "%-${width}s- %s\n" " {type}" "{power or temp} To check either power or temperatures" echo -e "\n ${idsCL[LightCyan]}power${idsCL[Default]} {flag} options" printf "%-${width}s- %s\n" " -a,-avg {#days}" "Show average values for X number of days" printf "%-${width}s- %s\n" " -e,-esxi" "Narrow to ESXi Hosts" printf "%-${width}s- %s\n" " -m,-mainsite" "Narrow to Mainsite Equipment" printf "%-${width}s- %s\n" " -o,-offsite" "Narrow to Offsite Equipment" printf "%-${width}s- %s\n" " -p,-power" "Narrow to Power Equipment" echo -e "\n ${idsCL[LightCyan]}temp${idsCL[Default]} {flag} options" printf "%-${width}s- %s\n" " -a,-avg {#days}" "Show average values for X number of days" printf "%-${width}s- %s\n" " -e,-esxi" "Narrow to ESXi Hosts Only" printf "%-${width}s- %s\n" " -f,-fans" "Display ESXi Host Fanspeed Info" printf "%-${width}s- %s\n" " -m,-mainsite" "Narrow to Mainsite Equipment" printf "%-${width}s- %s\n" " -o,-offsite" "Narrow to Offsite Equipment" printf "%-${width}s- %s\n" " -s,-search" "Narrow to search of sensor name" printf "%-${width}s- %s\n" " -S,-sensors" "Narrow to Sensors Only" echo -e "}\n" exit 0 ;; esac ;; test) SHUTDOWN_SERVER ${2} -esxi off & ;; iscsi-vm-shutdown) if [ "${2^^}" == "PRI" ] || [ "${2^^}" == "" ]; then ISCSIVMSHUTDOWN elif [ "${2^^}" == "SYS" ]; then ISCSISYSSHUTDOWN elif [ "${2^^}" == "ALL" ]; then ISCSIALLSHUTDOWN else echo -e "\niSCSI VM slection not valid\n" fi ;; shutdown_server) SHUTDOWN_SERVER ${2} ${3} ${4} ${5};; hostpower) HOSTPOWER ${2} ${3};; host) HOSTMGMT ${2} ${3} ${4} ${5} ${6} ${7};; set-drs) ${PW_FOLDER}/esxi-scripts/set-drs.ps1 ${2};; set-fanspeed) ESXI_FANSPEED_MGMT ${2} ${3} ${4} ${5};; pdu) PDUCONTROL ${2} ${3};; get-snapshots) GET_SNAPSHOTS;; service) if [ "${2}" == "status" ] || [ "${3}" == "status" ]; then CHECK_SERVICES elif [ "${2}" == "all" ]; then if [ "${3}" == "stop" ]; then STOPALL_SERVICES elif [ "${3}" == "start" ]; then STARTALL_SERVICES elif [ "${3}" == "restart" ]; then RESTARTALL_SERVICES else echo -e "\n${idsCL[Yellow]}Invalid argument${idsCL[Default]}\n" fi else if [ "${3}" == "start" ] || [ "${3}" == "" ]; then echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[${2}]}' Service Started" >> ${PW_LOGFILE} if [ "${2}" == "temp" ]; then # SENDNOTICE "Temp Monitor Startup" "$(DAILYTEMP)" & CHECKTEMP_SERVICE elif [ "${2}" == "power" ]; then CHECKPOWER_SERVICE elif [ "${2}" == "offsitepower" ]; then OFFSITE_CHECKPOWER_SERVICE elif [ "${2}" == "powerlogger" ]; then POWERLOGGER_SERVICE else echo -e "\n${idsCL[Yellow]}Invalid argument${idsCL[Default]}\n" fi elif [ "${3}" == "stop" ]; then if [ "${2}" == "temp" ]; then STOP_SERVICE temp echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[temp]}' Service Stopped" >> ${PW_LOGFILE} for SENSOR in ${!PW_SENSOR_ID[@]}; do logtemp=${PW_LOGFOLDER}/log-temp-${SENSOR} echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp} done elif [ "${2}" == "power" ]; then STOP_SERVICE power echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[power]}' Service Stopped" >> ${PW_LOGFILE} elif [ "${2}" == "offsitepower" ]; then STOP_SERVICE offsitepower echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[power]}' Service Stopped" >> ${PW_LOGFILE} elif [ "${2}" == "powerlogger" ]; then STOP_SERVICE powerlogger echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${PW_LOGPOWER} echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[powerlogger]}' Service Stopped" >> ${PW_LOGFILE} fi elif [ "${3}" == "restart" ]; then echo -e "($(date +'%Y-%m-%d %H:%M:%S')) - '${PW_POWERWALL_SERVICES[${2}]}' Service Restarted" >> ${PW_LOGFILE} echo SERVICE_MNG ${2} stop echo -en "${idsCL[Yellow]}Waiting${idsCL[Default]}" sleep 6s echo -en "\033[2K\r" DIVIDER . yellow 50 SERVICE_MNG ${2} start echo elif [ "${3}" == "status" ]; then srvc=${PW_POWERWALL_SERVICES[${2}]} c=0; spc=''; spc1=`expr 25 - ${#srvc}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; ((c++)); done echo -e "${idsCL[LightCyan]}${srvc}${spc}: " [ "$(systemctl is-active ${srvc})" == "active" ] && echo -en "${idsCL[Green]}Running" || echo -en "${idsCL[Yellow]}Not Running" echo -e "${idsCL[Default]}" echo fi fi ;; logrotate) LOGROTATE;; confsync) CONFSYNC;; status) CHECKPOWER -p ${2} ${3} ${4} ${5} CHECKTEMP -S ${2} ${3} ${4} ${5} CHECK_SERVICES ;; tempdiff) SERVERROOM_TEMP_DIFFERENCE ${2} ${3} ${4};; average) AVERAGETEMP ${2} ${3} ${4};; cooldown) COOLDOWN_SRVRM;; tinytuya-update) UPDATE_TINYTUYA_SERVER;; sendtemps) SENDTEMPS "TEST SENDTEMPS";; sendtemp) SENDTEMP;; dl|downlist) DOWNLIST;; oldhelp) echo -e "${idsCL[White]} Usage: ${idsCL[LightYellow]}[powerwall or pw]${idsCL[LightCyan]} {option}" echo echo -e " ${idsCL[LightCyan]}status ${idsCL[Default]}--> PowerWall Status Information" echo echo -e " ${idsCL[LightCyan]}check${idsCL[Yellow]} {type} {opts} ${idsCL[Default]}--> Check Sensors" echo -e " ${idsCL[Yellow]} type = { power / temp }" echo -e " ${idsCL[Yellow]} opts =" echo -e " ${idsCL[Yellow]} power { all / mainsite / offsite } ${idsCL[Default]}--> Defaults to All" echo -e " ${idsCL[Yellow]} temp { avg for # of days to show } ${idsCL[Default]}--> Defaults to not showing averages" echo echo -e " ${idsCL[LightCyan]}host${idsCL[Yellow]} {host} {flags} ${idsCL[Default]}--> Host Power/ESXi Control" echo -e " ${idsCL[Yellow]} host = individual {esxi-ip}, {idrac-ip}, {hostname} or {host-number;0-9}" echo -e " ${idsCL[Yellow]} host = groups { all / mainsite / offsite / main / sys }" echo -e " ${idsCL[Yellow]} flags =" echo -e " ${idsCL[Yellow]} -p,-power { on / off } ${idsCL[Default]}--> Controls Host Power" echo -e " ${idsCL[Yellow]} -e,-esxi { off / mm } ${idsCL[Default]}--> Shuts down ESXi or puts into maintenance mode" echo echo -e " ${idsCL[LightCyan]}iscsi-vm-shutdown${idsCL[Yellow]} {type} ${idsCL[Default]}--> Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)" echo -e " ${idsCL[Yellow]} type = { sys / main }" echo -e " ${idsCL[Yellow]} sys='iSCSI-SYS' / main='iSCSI-PRI' ${idsCL[Default]}--> Defaults to main (iSCSI-PRI)" echo echo -e " ${idsCL[LightCyan]}service${idsCL[Yellow]} {srvc} {cmd} ${idsCL[Default]}--> Manage an individual monitor service" echo -e " ${idsCL[Yellow]} srvc = { all / temp / power / powerlogger }" echo -e " ${idsCL[Yellow]} cmd = { start / stop / restart / status }" echo echo -e " ${idsCL[LightCyan]}update ${idsCL[Default]}--> Check for and install new PowerWall Script updates" echo ;; *) echo -e "\n${idsCL[Yellow]}Commands followed by '-h' will show specific command help. {work-in-progress}${idsCL[Default]}" echo echo -e "Usage: ${idsCL[LightCyan]}[powerwall or pw] ${idsCL[Yellow]}{COMMAND} ${idsCL[LightYellow]}{FLAGS}${idsCL[Default]} {" echo echo -e " ${idsCL[Yellow]}status${idsCL[Default]} - PowerWall Status Information" echo -e " ${idsCL[Yellow]}check${idsCL[Default]} - Check Power or Temperature Sensors" echo echo -e " ${idsCL[Yellow]}host${idsCL[Default]} - Host Power/ESXi Control" echo -e " ${idsCL[Yellow]}iscsi-vm-shutdown${idsCL[Default]} - Shutdown VM's running on iSCSI-(PRI or SYS) Datastore(s)" echo echo -e " ${idsCL[Yellow]}service${idsCL[Default]} - Manage an individual monitor service across multiple nodes" echo echo -e " ${idsCL[Yellow]}update${idsCL[Default]} - Check for and install new PowerWall Script updates" echo echo -e "}" echo echo exit 0 ;; esac # fi exit 0