This commit is contained in:
2023-05-16 20:15:28 -05:00
parent 0c093e9958
commit 2931ddc4c3
2 changed files with 66 additions and 24 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash
VERS='2.151-05152023'
VERS='2.155-05152023'
noheader=' update service '
@@ -17,8 +17,10 @@ min_battvolt_sys=47
FOLDER='/opt/idssys/powerwall'
LOGFOLDER=${FOLDER}/logs
LOGARCHIVE=${FOLDER}/logs/archive
TMPFOLDER=${FOLDER}/.tmp
[ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER}
[ ! -d ${LOGARCHIVE} ] && mkdir ${LOGARCHIVE}
[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
logfile=${LOGFOLDER}/logfile
logpower=${LOGFOLDER}/log-power

View File

@@ -129,9 +129,8 @@ Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR}
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1
echo -e "($datetime) - ${temp_f}'F - Main servers have been shutdown" >> ${logtemp}
SHUTDOWN_MAIN ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
@@ -142,10 +141,9 @@ Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP
SHUTDOWN_CRIT ${SENSOR}
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1
echo -e "($datetime) - ${temp_f}'F - System servers have been shutdown" >> ${logtemp}
SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} &
echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
fi
else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
@@ -169,12 +167,17 @@ NORMAL TEMP: ${temp_f}'F"
fi
last_temp[${SENSOR}]=1
fi
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
[ -f ${TMPFOLDER}/${SENSOR}.insert ] && errtime=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || errtime=0
if [ ${errtime} -ge 300 ]; then
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${TMPFOLDER}/${SENSOR}.insert
fi
${mysql_conn} -e "${QRY}"
else
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
echo "($datetime) - alert sent" >> ${logtemp}
@@ -239,6 +242,7 @@ CHECKPOWER_SERVICE(){
fi
fi
voltstatus=1
rm -f ${TMPFOLDER}/power.ac.*
else
echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logpower}
echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logfile}
@@ -250,29 +254,33 @@ CHECKPOWER_SERVICE(){
last_battvolt=$BATTVOLT
fi
voltstatus=3
touch ${TMPFOLDER}/power.ac.low
fi
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
Shutting down main servers" 1
SHUTDOWN_MAIN server
SHUTDOWN_MAIN power &
battstatus=2
fi
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
Shutting down all servers" 1
SHUTDOWN_SYS server
SHUTDOWN_SYS power &
battstatus=3
fi
else
@@ -281,7 +289,7 @@ CHECKPOWER_SERVICE(){
if [ $battstatus -eq 0 ]; then
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logpower}
fi
rm -f ${TMPFOLDER}/power.dc.*
battstatus=1
fi
@@ -320,9 +328,14 @@ POWERLOGGER_SERVICE(){
${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}"
sleep 1m
if [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
sleep 1m
else
sleep 5m
fi
done
}
@@ -381,12 +394,38 @@ CHECK_SERVICES(){
}
SHUTDOWN_MAIN(){
SHUTDOWN_SERVERS MAIN
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS MAIN ${2}
reason='Server Room Overheated!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown
${reason}" 1
echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${logfile}
fi
echo
}
SHUTDOWN_SYS(){
SHUTDOWN_SERVERS SYS
curl --data "cmd=PWRoff" http://10.10.0.61/cmd
if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS SYS ${2}
reason='Server Room Overheated!'
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown
${reason}" 1
echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${logfile}
fi
# curl --data "cmd=PWRoff" http://10.10.0.61/cmd
echo
}
SHUTDOWN_CRIT(){
@@ -592,7 +631,8 @@ fi
;;
checktemp) CHECKTEMP ${2};;
checkpower) CHECKPOWER;;
test) LOGROTATE ${2};;
test) TEST ${2};;
logrotate) LOGROTATE;;
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
shutdownhost) SHUTDOWN_SERVER ${2};;