524 lines
19 KiB
Bash
Executable File
524 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
action="${1}"
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/powerwall/defaults.inc
|
|
source ${FOLDER}/powerwall.conf
|
|
|
|
|
|
shopt -s lastpipe
|
|
|
|
VERBOSE=false
|
|
|
|
|
|
CHECKTEMP(){
|
|
cw=20;
|
|
echo
|
|
echo -e "${idsCL[Yellow]}Sensor Temperature(s) ${idsCL[Green]}normal${idsCL[Default]} /${idsCL[Yellow]}warning${idsCL[Default]}/ ${idsCL[Red]}critical${idsCL[Default]}"
|
|
DIVIDER . yellow 75
|
|
if [ "${1}" != "" ]; then
|
|
# reading=$(CHECKTEMPSENSOR ${1})
|
|
# c=0; spc=''; spc1=`expr ${cw} - ${#1}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
# echo -e "${idsCL[LightCyan]}${1}${spc}${idsCL[Default]}: ${reading}"
|
|
|
|
for SENSOR in ${!SENSOR_ID[@]}; do
|
|
if [[ "${SENSOR}" = *"${1}"* ]]; then
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR})
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
|
|
fi
|
|
done
|
|
|
|
else
|
|
for SENSOR in ${!SENSOR_ID[@]}; do
|
|
reading=$(CHECKTEMPSENSOR ${SENSOR})
|
|
c=0; spc=''; spc1=`expr ${cw} - ${#SENSOR}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "${idsCL[LightCyan]}${SENSOR}${spc}${idsCL[Default]}: ${reading}"
|
|
done
|
|
fi
|
|
echo
|
|
}
|
|
CHECKTEMPSENSOR(){
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${1}]} | cut -d',' -f 2 | read temp_crit
|
|
|
|
if [ "${SENSOR_TYPE[${1}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${1}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${SENSOR_TYPE[${1}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
|
|
fi
|
|
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
#echo "$temp_c -> $temp_f"
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
echo -en "${idsCL[Yellow]}WARNING ($temp_f'F)${idsCL[Default]}"
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ]; then
|
|
echo -en "${idsCL[Red]}CRITICAL ($temp_f'F)${idsCL[Default]}"
|
|
else
|
|
echo -en "${idsCL[Green]}Normal ($temp_f'F)${idsCL[Default]}"
|
|
fi
|
|
|
|
c=0; spc=''; spc1=`expr 13 - ${#temp_f} - 2`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
[ ${#temp_warn} == 2 ] && spcw=' ' || spcw=' '
|
|
[ ${#temp_crit} == 2 ] && spcc=' ' || spcc=' '
|
|
echo -e "${spc}[${spcw}${idsCL[Green]}<=$(expr ${temp_warn} - 1)'F${idsCL[Default]} /${spcw}${idsCL[Yellow]}${temp_warn}'F${idsCL[Default]} /${spcc}${idsCL[Red]}${temp_crit}'F${idsCL[Default]} ]"
|
|
fi
|
|
}
|
|
|
|
CHECKTEMP_SERVICE(){
|
|
cw=20
|
|
declare -A last_temp
|
|
while true; do
|
|
|
|
for SENSOR in ${!SENSOR_ID[@]}; do
|
|
logtemp=${LOGFOLDER}/log-temp-${SENSOR}
|
|
[ ! -f ${logtemp} ] && touch ${logtemp}
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logtemp}))
|
|
[ $errtime -ge 3600 ] && relog=1 || relog=0
|
|
|
|
if [ "${SENSOR_TYPE[${SENSOR}]}" == "mqtt" ]; then
|
|
mqtt_message=`${mqtt_conn} -t tele/${SENSOR}/SENSOR -C 1`
|
|
echo $mqtt_message | cut -d':' -f 7 | cut -d',' -f 1 | read temp_h
|
|
echo $mqtt_message | cut -d':' -f 6 | cut -d',' -f 1 | read temp_c
|
|
|
|
elif [ "${SENSOR_TYPE[${SENSOR}]}" == "system" ]; then
|
|
temp_c=$(vcgencmd measure_temp)
|
|
temp_c=${temp_c%\'*}; temp_c=${temp_c#*=}
|
|
temp_h=''
|
|
fi
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
if [ "$temp_c" != "null" ] && [ "$temp_c" != "" ]; then
|
|
temp_f=`echo "scale=2; $temp_c*1.8 + 32" | bc`
|
|
|
|
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 1 | read temp_warn
|
|
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 2 | read temp_crit
|
|
echo ${TEMP_THRESHOLDS[${SENSOR}]} | cut -d',' -f 3 | read temp_crit_sys
|
|
|
|
if [ "${last_temp[${SENSOR}]}" != "" ]; then
|
|
temp_diff=`expr ${last_temp[${SENSOR}]%.*} - ${temp_f%.*}`
|
|
[ $temp_diff -lt 0 ] && temp_diff=$(($temp_diff * -1))
|
|
else
|
|
last_temp[${SENSOR}]=0
|
|
temp_diff=0
|
|
fi
|
|
|
|
if [ $(bc -l <<< "$temp_f >= $temp_warn") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit") -eq 1 ]; then
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP WARNING" "($datetime) WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" 1
|
|
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
|
|
last_temp[${SENSOR}]=$temp_f
|
|
fi
|
|
echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
|
|
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
|
|
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
|
|
Shutting down servers!!" 1
|
|
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
|
|
last_temp[${SENSOR}]=$temp_f
|
|
SHUTDOWN_MAIN ${SENSOR}
|
|
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
|
|
echo -e "($datetime) - ${SENSOR} - ${temp_f}F - Main servers have been shutdown" >> ${logtemp}
|
|
|
|
fi
|
|
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
|
|
echo "($datetime) - ${temp_f}F - SYSTEM CRITICAL - (Difference of ${temp_diff})" >> ${logtemp}
|
|
if [ $temp_diff -gt 1 ] || [ "${last_temp[${SENSOR}]}" = "1" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
|
|
Shutting down system servers!!" 1
|
|
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
|
|
last_temp[${SENSOR}]=$temp_f
|
|
SHUTDOWN_SYS ${SENSOR}
|
|
SHUTDOWN_CRIT ${SENSOR}
|
|
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
|
|
echo -e "($datetime) - ${SENSOR} - ${temp_f}F - System servers have been shutdown" >> ${logtemp}
|
|
fi
|
|
else
|
|
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
|
|
SENDNOTICE "${SENSOR} BACK TO NORMAL" "($datetime) NORMAL TEMP: ${temp_f}'F
|
|
Previous Temp: ${last_temp[${SENSOR}]}'F"
|
|
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
|
|
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
|
|
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
|
|
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
|
|
NORMAL TEMP: ${temp_f}'F"
|
|
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
|
|
fi
|
|
last_temp[${SENSOR}]=1
|
|
[ $relog -eq 1 ] && echo "($datetime) - ${SENSOR} - ${temp_f}F - Normal Temp" >> ${logtemp}
|
|
fi
|
|
if [ "${temp_h}" != "" ]; then
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
|
|
else
|
|
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
|
|
fi
|
|
${mysql_conn} -e "${QRY}"
|
|
else
|
|
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
|
|
echo "($datetime) - ${SENSOR} - alert sent" >> ${logtemp}
|
|
fi
|
|
|
|
done
|
|
|
|
sleep 10
|
|
|
|
done # &
|
|
}
|
|
|
|
CHECKPOWER(){
|
|
JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
|
|
INPUTV=`echo ${JSONSTATS} | jq '.inputs .inV'`
|
|
INPUTA=`echo ${JSONSTATS} | jq '.inputs .inA'`
|
|
OUTPUTV=`echo ${JSONSTATS} | jq '.outputs .outV'`
|
|
OUTPUTA=`echo ${JSONSTATS} | jq '.outputs .outA'`
|
|
BATTV=`echo ${JSONSTATS} | jq '.inputs .battV'`
|
|
BATTA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
|
|
|
|
OUTPUTV2=$(awk -vp_val="$OUTPUTV" 'BEGIN{print p_val/2}')
|
|
OUTPUTA2=$(awk -vp_val="$OUTPUTA" 'BEGIN{print p_val*2}')
|
|
|
|
echo
|
|
echo -e "${idsCL[Yellow]}Power Information"
|
|
DIVIDER . yellow 75
|
|
|
|
echo -e "${idsCL[LightCyan]}Input Power ${idsCL[Default]}: ${idsCL[Green]}${INPUTV}V / ${INPUTA}A"
|
|
echo -e "${idsCL[LightCyan]}Output Power ${idsCL[Default]}: ${idsCL[Green]}${OUTPUTV}V (${OUTPUTV2}V) / ${OUTPUTA}A (${OUTPUTA2}A @ ${OUTPUTV2}V)"
|
|
echo -e "${idsCL[LightCyan]}Battery Power ${idsCL[Default]}: ${idsCL[Green]}${BATTV}V / ${BATTA}A ${idsCL[Default]}"
|
|
[ "${action}" != "" ] && echo
|
|
}
|
|
|
|
CHECKACV_SERVICE(){
|
|
voltstatus=0
|
|
battstatus=0
|
|
while true; do
|
|
INPUTACV=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .inV')
|
|
BATTVOLT=$(curl -s "http://10.10.0.61/stats.json" | jq '.inputs .battV')
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
if [ ${INPUTACV} -ge ${min_acvolt} ]; then
|
|
if [ $voltstatus -eq 3 ]; then
|
|
echo "($datetime) - ${INPUTACV}V - Back to Normal Voltage" >> ${logacv}
|
|
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}V"
|
|
echo "($datetime) alert sent" >> ${logacv}
|
|
battstatus=0
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${INPUTACV}V - Normal Voltage" >> ${logacv}
|
|
if [ $voltstatus -eq 0 ]; then
|
|
SENDNOTICE "Power Nominal" "($datetime) Service Startup
|
|
Normal voltage detected
|
|
VOLTAGE: ${INPUTACV}V
|
|
BATT VOLTAGE: ${BATTVOLT}V"
|
|
echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
|
|
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logacv}
|
|
fi
|
|
fi
|
|
voltstatus=1
|
|
else
|
|
echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
|
|
if [ $voltstatus -lt 3 ]; then
|
|
SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected
|
|
VOLTAGE: ${INPUTACV}V
|
|
BATT VOLTAGE: ${BATTVOLT}V" 1
|
|
echo "($datetime) alert sent" >> ${logacv}
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
voltstatus=3
|
|
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
|
|
if [ $battstatus -lt 2 ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
|
|
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
|
|
Shutting down main servers" 1
|
|
SHUTDOWN_MAIN server
|
|
battstatus=2
|
|
fi
|
|
|
|
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
|
|
if [ $battstatus -lt 3 ]; then
|
|
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
|
|
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
|
|
Shutting down all servers" 1
|
|
SHUTDOWN_SYS server
|
|
|
|
battstatus=3
|
|
fi
|
|
|
|
else
|
|
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
|
|
[ $errtime -ge 3600 ] && echo "($datetime) - ${BATTVOLT}V - Battery Voltage" >> ${logacv}
|
|
|
|
battstatus=1
|
|
|
|
fi
|
|
volt_diff=$((${last_battvolt%.*} - ${BATTVOLT%.*}))
|
|
[ $volt_diff -lt 0 ] && temp_diff=$(($volt_diff * -1))
|
|
if [ $volt_diff -gt 0 ]; then
|
|
SENDNOTICE "BATTERY VOLTAGE CHANGE" "($datetime) Battery Voltage: ${BATTVOLT}V"
|
|
echo -e "($datetime) - ${BATTVOLT}V - Battery Voltage Change" >> ${logacv}
|
|
last_battvolt=$BATTVOLT
|
|
fi
|
|
fi
|
|
|
|
sleep 10
|
|
done # &
|
|
}
|
|
|
|
POWERLOGGER_SERVICE(){
|
|
while true; do
|
|
datetime=`date +'%Y-%m-%d %H:%M:%S'`
|
|
|
|
JSONSTATS=$(curl -s "http://10.10.0.61/stats.json")
|
|
inV=`echo ${JSONSTATS} | jq '.inputs .inV'`
|
|
inA=`echo ${JSONSTATS} | jq '.inputs .inA'`
|
|
outV=`echo ${JSONSTATS} | jq '.outputs .outV'`
|
|
outA=`echo ${JSONSTATS} | jq '.outputs .outA'`
|
|
battV=`echo ${JSONSTATS} | jq '.inputs .battV'`
|
|
xfA=`echo ${JSONSTATS} | jq '.inputs .xfA'`
|
|
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('3','${datetime}','${inV}','${inA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('4','${datetime}','${outV}','${outA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO power_data (\`sensorid\`, \`date\`, \`volt\`, \`amp\`) VALUES ('5','${datetime}','${battV}','${xfA}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
|
|
${mysql_conn} -e "USE servermonitor; ${QRY}"
|
|
|
|
sleep 1m
|
|
|
|
done
|
|
}
|
|
|
|
RESTART_SERVICES(){
|
|
STOPALL_SERVICES
|
|
sleep 5s
|
|
STARTALL_SERVICES
|
|
}
|
|
STOPALL_SERVICES(){
|
|
for srvc in "${POWERWALL_SERVICES[@]}"; do
|
|
echo "Stopping $srvc"
|
|
/bin/systemctl stop $srvc
|
|
done
|
|
}
|
|
STARTALL_SERVICES(){
|
|
for srvc in "${POWERWALL_SERVICES[@]}"; do
|
|
echo "Starting $srvc"
|
|
/bin/systemctl start $srvc
|
|
done
|
|
}
|
|
|
|
STOP_SERVICE(){
|
|
echo "stop: ${1}"
|
|
service_pid=`systemctl show --property MainPID --value ${POWERWALL_SERVICES[${1}]}`
|
|
/usr/bin/pkill -P $service_pid
|
|
}
|
|
|
|
SHUTDOWN_MAIN(){
|
|
SHUTDOWN_SERVERS MAIN
|
|
echo
|
|
}
|
|
SHUTDOWN_SYS(){
|
|
SHUTDOWN_SERVERS SYS
|
|
curl --data "cmd=PWRoff" http://10.10.0.61/cmd
|
|
echo
|
|
}
|
|
SHUTDOWN_CRIT(){
|
|
if [ "${1}" == "RaspberryPI-CPU" ]; then
|
|
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) Powerwall System shutting down" 1
|
|
echo -e "($datetime) - ${SENSOR} - CRITICAL - Powerwall System shutting down" >> ${logtemp}
|
|
/sbin/poweroff
|
|
fi
|
|
}
|
|
|
|
SHUTDOWN_SERVERS(){
|
|
touch $FOLDER/shutdown
|
|
|
|
# HOSTS=${1}HOSTS[@]
|
|
# for ESXIHOST in "${!HOSTS}"; do
|
|
# echo "Shutting Down: ${ESXIHOST}"
|
|
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "cp /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown.sh /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/esxidown-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "sed -i 's/\$SCRIPTPATH\/esxidown.sh/\$SCRIPTPATH\/esxidown-${ESXIHOST}.sh/g' /vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# /usr/bin/ssh root@${ESXIHOST} "/vmfs/volumes/iSCSI2-Datastore2\ \(R5\)\(1-4\)/\!SCRIPTS/esxidown/async-${ESXIHOST}.sh"
|
|
# done
|
|
|
|
|
|
if [ "${1}" = "MAIN" ]; then
|
|
echo -en "Disabling CLS in vCenter ... "
|
|
ssh root@${VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
|
|
ssh root@${VCENTERHOST} "service-control --stop vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
|
|
ssh root@${VCENTERHOST} "service-control --start vmware-vpxd >/dev/null 2>&1" >/dev/null 2>&1
|
|
echo "Done"
|
|
|
|
for ESXIHOST in "${ESXIHOSTS[@]}"; do
|
|
echo -en "Stopping iSCSI VM's on: ${ESXIHOST} ... "
|
|
${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
for ESXIHOST in "${ESXIHOSTS[@]}"; do
|
|
echo -en "Verifying iSCSI VM's stopped on: ${ESXIHOST} ... "
|
|
${FOLDER}/esxi-scripts/iscsi-vm-shutdown-verify.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
# sleep 1m
|
|
for ESXIHOST in "${MAINHOSTS[@]}"; do
|
|
echo -en "Stopping Remaining VM's on: ${ESXIHOST} ... "
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0' &
|
|
${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
sleep 1m
|
|
for ESXIHOST in "${MAINHOSTS[@]}"; do
|
|
echo -en "Shutting down host: ${ESXIHOST} ..."
|
|
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"'
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
|
|
echo "Done"
|
|
done
|
|
|
|
for ESXIHOST in "${MAINHOSTS[@]}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Done"
|
|
done
|
|
|
|
elif [ "${1}" = "SYS" ]; then
|
|
for ESXIHOST in "${SYSHOSTS[@]}"; do
|
|
echo -en "Shutting Down VM's on: ${ESXIHOST} ... "
|
|
${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo "Done"
|
|
done
|
|
for ESXIHOST in "${SYSHOSTS[@]}"; do
|
|
echo -en "Shutting Down Host: ${ESXIHOST} ... "
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e true -t 0'
|
|
ssh root@${ESXIHOST} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
|
|
ssh root@${ESXIHOST} 'esxcli system maintenanceMode set -e false -t 0'
|
|
echo "Done"
|
|
done
|
|
|
|
for ESXIHOST in "${SYSHOSTS[@]}"; do
|
|
echo -en "Waiting for Host (${ESXIHOST}) to shutdown ... "
|
|
while ping -qw 10 -c3 "${ESXIHOST}">/dev/null 2>&1; do
|
|
sleep 1
|
|
done
|
|
echo "Done"
|
|
done
|
|
|
|
fi
|
|
}
|
|
SHUTDOWN_SERVER(){
|
|
echo -en "Enabling Maintenance mode and will wait for iSCSI VM's to begin migrating ... "
|
|
${FOLDER}/esxi-scripts/host-maintenance_modce.ps1 ${ESXI_HOST_NAMES[${1}]}
|
|
# sleep 10s
|
|
echo "Done"
|
|
|
|
# echo -en "Shutting Down VM's on: ${1} ... "
|
|
# ${FOLDER}/esxi-scripts/vm-shutdown.ps1 ${1} >/dev/null 2>&1
|
|
# ${FOLDER}/esxi-scripts/iscsi-server-shutdown.ps1 ${1} >/dev/null 2>&1
|
|
# echo "Done"
|
|
#
|
|
# echo -en "Shutting Down Host: ${1} ... "
|
|
# ssh root@${1} 'esxcli system shutdown poweroff -d 10 -r "Automated ESXi host shutdown"' &
|
|
# ssh root@${1} 'esxcli system maintenanceMode set -e false -t 0'
|
|
# echo "Done"
|
|
#
|
|
# echo -en "Waiting for Host (${1}) to shutdown ... "
|
|
# while ping -qw 10 -c3 "${1}">/dev/null 2>&1; do
|
|
# sleep 1
|
|
# done
|
|
# echo "Done"
|
|
}
|
|
|
|
ISCSIVMSHUTDOWN(){
|
|
for ESXIHOST in "${ESXIHOSTS[@]}"; do
|
|
echo -en "${idsCL[LightCyan]}Shutting down any VM's on '${ESXIHOST}, that are running on an iSCSI drive ... "
|
|
# ${FOLDER}/esxi-scripts/iscsi-vm-shutdown.ps1 ${ESXIHOST} >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}DONE${idsCL[Default]}"
|
|
echo
|
|
done
|
|
|
|
}
|
|
|
|
TEST(){
|
|
# echo "Getting info from ${1}"
|
|
# ${FOLDER}/esxi-scripts/sys-iscsi-shutdown.ps1 ${1}
|
|
|
|
for ESXIHOST in "${ESXIHOSTS[@]}"; do
|
|
ssh-copy-id root@${ESXIHOST}
|
|
done
|
|
|
|
}
|
|
|
|
|
|
if [[ "${1}" != *"_service" ]] && [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
|
|
echo
|
|
DIVIDER . lightGreen 50
|
|
c=0; spc=''; spc1=`expr 18 - ${#VERS}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -e "${idsCL[LightGreen]}| PowerWall Management ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${spc}${idsCL[LightGreen]}|${idsCL[Default]}"
|
|
DIVIDER . lightGreen 50
|
|
fi
|
|
|
|
# if [ ${action-x} ]; then
|
|
case $action in
|
|
stopall) STOPALL_SERVICES;;
|
|
startall) STARTALL_SERVICES;;
|
|
restartall) RESTART_SERVICES;;
|
|
checktemp) CHECKTEMP ${2};;
|
|
checkpower) CHECKPOWER;;
|
|
test) TEST ${2};;
|
|
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
|
|
|
|
shutdownhost) SHUTDOWN_SERVER ${2};;
|
|
shutdown_servers) SHUTDOWN_SERVERS ${2};;
|
|
|
|
checktemp_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE Temp
|
|
for SENSOR in ${!SENSOR_ID[@]}; do
|
|
logtemp=${LOGFOLDER}/log-temp-${SENSOR}
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) - ${SENSOR} - Service Stopped" >> ${logtemp}
|
|
done
|
|
else
|
|
CHECKTEMP_SERVICE
|
|
fi
|
|
;;
|
|
checkacv_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE ACV
|
|
echo "(`date +'%Y-%m-%d %H:%M:%S'`) Service Stopped" >> ${logacv}
|
|
else
|
|
CHECKACV_SERVICE
|
|
fi
|
|
;;
|
|
powerlogger_service)
|
|
if [ "${2}" = "stop" ]; then
|
|
STOP_SERVICE powerlogger
|
|
else
|
|
POWERLOGGER_SERVICE
|
|
fi
|
|
;;
|
|
update);;
|
|
*)
|
|
CHECKPOWER
|
|
CHECKTEMP
|
|
;;
|
|
esac
|
|
# fi
|
|
|
|
exit 0 |