This commit is contained in:
2023-05-16 20:15:28 -05:00
parent 0c093e9958
commit 2931ddc4c3
2 changed files with 66 additions and 24 deletions

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
VERS='2.151-05152023' VERS='2.155-05152023'
noheader=' update service ' noheader=' update service '
@@ -17,8 +17,10 @@ min_battvolt_sys=47
FOLDER='/opt/idssys/powerwall' FOLDER='/opt/idssys/powerwall'
LOGFOLDER=${FOLDER}/logs LOGFOLDER=${FOLDER}/logs
LOGARCHIVE=${FOLDER}/logs/archive LOGARCHIVE=${FOLDER}/logs/archive
TMPFOLDER=${FOLDER}/.tmp
[ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER} [ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER}
[ ! -d ${LOGARCHIVE} ] && mkdir ${LOGARCHIVE} [ ! -d ${LOGARCHIVE} ] && mkdir ${LOGARCHIVE}
[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER}
logfile=${LOGFOLDER}/logfile logfile=${LOGFOLDER}/logfile
logpower=${LOGFOLDER}/log-power logpower=${LOGFOLDER}/log-power

View File

@@ -129,9 +129,8 @@ Shutting down servers!!" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile} echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f last_temp[${SENSOR}]=$temp_f
SHUTDOWN_MAIN ${SENSOR} SHUTDOWN_MAIN ${SENSOR} &
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1 echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - Main servers have been shutdown" >> ${logtemp}
fi fi
elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then
@@ -142,10 +141,9 @@ Shutting down system servers!!" 1
echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile} echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile}
last_temp[${SENSOR}]=$temp_f last_temp[${SENSOR}]=$temp_f
SHUTDOWN_SYS ${SENSOR} TEMP SHUTDOWN_SYS ${SENSOR} TEMP &
SHUTDOWN_CRIT ${SENSOR} SHUTDOWN_CRIT ${SENSOR} &
SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1 echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp}
echo -e "($datetime) - ${temp_f}'F - System servers have been shutdown" >> ${logtemp}
fi fi
else else
if [ ${last_temp[${SENSOR}]} -gt 1 ]; then if [ ${last_temp[${SENSOR}]} -gt 1 ]; then
@@ -169,12 +167,17 @@ NORMAL TEMP: ${temp_f}'F"
fi fi
last_temp[${SENSOR}]=1 last_temp[${SENSOR}]=1
fi fi
if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" [ -f ${TMPFOLDER}/${SENSOR}.insert ] && errtime=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || errtime=0
else if [ ${errtime} -ge 300 ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" if [ "${temp_h}" != "" ]; then
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')"
else
QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')"
fi
${mysql_conn} -e "${QRY}"
touch ${TMPFOLDER}/${SENSOR}.insert
fi fi
${mysql_conn} -e "${QRY}"
else else
SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1 SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1
echo "($datetime) - alert sent" >> ${logtemp} echo "($datetime) - alert sent" >> ${logtemp}
@@ -239,6 +242,7 @@ CHECKPOWER_SERVICE(){
fi fi
fi fi
voltstatus=1 voltstatus=1
rm -f ${TMPFOLDER}/power.ac.*
else else
echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logpower} echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logpower}
echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logfile} echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logfile}
@@ -250,29 +254,33 @@ CHECKPOWER_SERVICE(){
last_battvolt=$BATTVOLT last_battvolt=$BATTVOLT
fi fi
voltstatus=3 voltstatus=3
touch ${TMPFOLDER}/power.ac.low
fi fi
if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower} echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile} echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logpower} echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V
Shutting down main servers" 1 Shutting down main servers" 1
SHUTDOWN_MAIN server SHUTDOWN_MAIN power &
battstatus=2 battstatus=2
fi fi
elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then
echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower} echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower}
echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile} echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile}
touch ${TMPFOLDER}/power.dc.low
if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then
echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logpower} echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logpower}
SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V
Shutting down all servers" 1 Shutting down all servers" 1
SHUTDOWN_SYS server SHUTDOWN_SYS power &
battstatus=3 battstatus=3
fi fi
else else
@@ -281,7 +289,7 @@ CHECKPOWER_SERVICE(){
if [ $battstatus -eq 0 ]; then if [ $battstatus -eq 0 ]; then
echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logpower} echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logpower}
fi fi
rm -f ${TMPFOLDER}/power.dc.*
battstatus=1 battstatus=1
fi fi
@@ -320,9 +328,14 @@ POWERLOGGER_SERVICE(){
${mysql_conn} -e "USE servermonitor; ${QRY}" ${mysql_conn} -e "USE servermonitor; ${QRY}"
QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')" QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')"
${mysql_conn} -e "USE servermonitor; ${QRY}" ${mysql_conn} -e "USE servermonitor; ${QRY}"
sleep 1m
if [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then
sleep 1m
else
sleep 5m
fi
done done
} }
@@ -381,12 +394,38 @@ CHECK_SERVICES(){
} }
SHUTDOWN_MAIN(){ SHUTDOWN_MAIN(){
SHUTDOWN_SERVERS MAIN if [ "${1}" == "ServerRoomTH"]; then
SHUTDOWN_SERVERS MAIN ${2}
reason='Server Room Overheated!'
sendnotice=true
else
sendnotice=false
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown
${reason}" 1
echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${logfile}
fi
echo echo
} }
SHUTDOWN_SYS(){ SHUTDOWN_SYS(){
SHUTDOWN_SERVERS SYS if [ "${1}" == "ServerRoomTH"]; then
curl --data "cmd=PWRoff" http://10.10.0.61/cmd SHUTDOWN_SERVERS SYS ${2}
reason='Server Room Overheated!'
fi
if [ "${sendnotice}" == "true"]; then
datetime=`date +'%Y-%m-%d %H:%M:%S'`
SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown
${reason}" 1
echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${logfile}
fi
# curl --data "cmd=PWRoff" http://10.10.0.61/cmd
echo echo
} }
SHUTDOWN_CRIT(){ SHUTDOWN_CRIT(){
@@ -592,7 +631,8 @@ fi
;; ;;
checktemp) CHECKTEMP ${2};; checktemp) CHECKTEMP ${2};;
checkpower) CHECKPOWER;; checkpower) CHECKPOWER;;
test) LOGROTATE ${2};; test) TEST ${2};;
logrotate) LOGROTATE;;
iscsi-vm-shutdown) ISCSIVMSHUTDOWN;; iscsi-vm-shutdown) ISCSIVMSHUTDOWN;;
shutdownhost) SHUTDOWN_SERVER ${2};; shutdownhost) SHUTDOWN_SERVER ${2};;