This commit is contained in:
2023-05-15 19:22:22 -05:00
parent d1c6a996bb
commit 7d6c61f531
2 changed files with 16 additions and 0 deletions

View File

@@ -23,6 +23,7 @@ logacv=${LOGFOLDER}/log-acv
declare -A TEMP_THRESHOLDS
# TEMP_THRESHOLDS['ServerRoomTH']="72,74,76"
TEMP_THRESHOLDS['ServerRoomTH']="83,88,95"
TEMP_THRESHOLDS['RaspberryPI-CPU']="122,131,176"

View File

@@ -120,6 +120,7 @@ CHECKTEMP_SERVICE(){
last_temp[${SENSOR}]=$temp_f
fi
echo "($datetime) - ${temp_f}F - WARNING TEMP - (Difference of ${temp_diff}')" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - WARNING TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" > ${logfile}
elif [ $(bc -l <<< "$temp_f >= $temp_crit") -eq 1 ] && [ $(bc -l <<< "$temp_f < $temp_crit_sys") -eq 1 ]; then
echo "($datetime) - ${temp_f}'F - CRITICAL - (Difference of ${temp_diff}'F)" >> ${logtemp}
@@ -127,6 +128,7 @@ CHECKTEMP_SERVICE(){
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down servers!!" 1
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')" > ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR}
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
@@ -139,6 +141,7 @@ Shutting down servers!!" 1
SENDNOTICE "${SENSOR} TEMP CRITICAL" "($datetime) SYSTEM CRITICAL TEMP: ${temp_f}'F - (Difference of ${temp_diff}')
Shutting down system servers!!" 1
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - CRITICAL TEMP SYSTEM: ${temp_f}'F - (Difference of ${temp_diff}')" > ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP
SHUTDOWN_CRIT ${SENSOR}
@@ -152,11 +155,14 @@ Previous Temp: ${last_temp[${SENSOR}]}'F"
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Back to NORMAL TEMP - Previous Temp: ${last_temp[${SENSOR}]}'F" >> ${logtemp}
echo "($datetime) - ${SENSOR} - ${temp_f}F - alert sent" >> ${logtemp}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
echo -e "($datetime) - ${SENSOR} - Back to NORMAL TEMP: ${temp_f}'F" > ${logfile}
elif [ ${last_temp[${SENSOR}]} -eq 0 ]; then
SENDNOTICE "${SENSOR} TEMP NORMAL" "($datetime) Service Startup
NORMAL TEMP: ${temp_f}'F"
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - Service Startup - NORMAL TEMP" >> ${logtemp}
[ "$(compgen -G "${FOLDER}/shutdown.*")" != "" ] && rm -f ${FOLDER}/shutdown.*
echo -e "($datetime) - ${SENSOR} - Service Startup: ${temp_f}'F" > ${logfile}
fi
last_temp[${SENSOR}]=1
[ $relog -eq 1 ] && echo "($datetime) - ${SENSOR} - ${temp_f}F - Normal Temp" >> ${logtemp}
@@ -215,6 +221,7 @@ CHECKACV_SERVICE(){
SENDNOTICE "POWER NOMINAL AGAIN" "($datetime) Normal voltage detected
VOLTAGE: ${INPUTACV}V"
echo "($datetime) alert sent" >> ${logacv}
echo -e "($datetime) - ${INPUTACV}V - POWER NOMINAL AGAIN" > ${logfile}
battstatus=0
else
errtime=$(expr `date +%s` - $(stat -c %Y ${logacv}))
@@ -226,11 +233,13 @@ VOLTAGE: ${INPUTACV}V
BATT VOLTAGE: ${BATTVOLT}V"
echo "($datetime) - Input: ${INPUTACV}V - Service Startup - Normal voltage detected" >> ${logacv}
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logacv}
echo -e "($datetime) - ${INPUTACV}V / ${BATTVOLT}V - Service Startup - Normal Voltage" > ${logfile}
fi
fi
voltstatus=1
else
echo "($datetime) - ${INPUTACV}V - LOW Voltage" >> ${logacv}
echo -e "($datetime) - ${INPUTACV}V - LOW VOLTAGE" > ${logfile}
if [ $voltstatus -lt 3 ]; then
SENDNOTICE "POWER ALERT - LOW POWER!!" "($datetime) Power off or low voltage detected
VOLTAGE: ${INPUTACV}V
@@ -241,6 +250,7 @@ BATT VOLTAGE: ${BATTVOLT}V" 1
voltstatus=3
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" > ${logfile}
if [ $battstatus -lt 2 ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logacv}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
@@ -251,6 +261,7 @@ Shutting down main servers" 1
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logacv}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" > ${logfile}
if [ $battstatus -lt 3 ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logacv}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
@@ -361,6 +372,7 @@ SHUTDOWN_SERVERS(){
if [ "${1}" = "MAIN" ]; then
touch $FOLDER/shutdown.main
echo "($datetime) - Shutting down Main servers" > ${logfile}
echo -en "Disabling CLS in vCenter ... "
if [ $TEST != true ]; then
ssh root@${VCENTERHOST} "sed -i '/<domain-c1001>/{n;s/<enabled>.*<\/enabled>/<enabled>False<\/enabled>/}' /etc/vmware-vpx/vpxd.cfg >/dev/null 2>&1" >/dev/null 2>&1
@@ -414,6 +426,7 @@ SHUTDOWN_SERVERS(){
SHUTDOWN_SERVERS MAIN
fi
[ $TEST != true ] && touch $FOLDER/shutdown.sys
echo "($datetime) - Shutting down System servers" > ${logfile}
for ESXIHOST in "${SYSHOSTS[@]}"; do
echo -en "Suspending DB, PiHole & vCenter VM's on: ${ESXIHOST} ... "
@@ -451,6 +464,8 @@ SHUTDOWN_SERVERS(){
fi
}
SHUTDOWN_SERVER(){
echo "($datetime) - Shutting down ${1}" > ${logfile}
echo -en "Enabling Maintenance mode and will wait for iSCSI VM's to begin migrating ... "
[ $TEST != true ] && ${FOLDER}/esxi-scripts/host-maintenance_mode.ps1 ${ESXI_HOST_NAMES[${1}]} & >/dev/null 2>&1
sleep 45s