diff --git a/defaults.inc b/defaults.inc index d2d095e0..3d3070be 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,6 +1,6 @@ #!/usr/bin/env bash -VERS='2.151-05152023' +VERS='2.155-05152023' noheader=' update service ' @@ -17,8 +17,10 @@ min_battvolt_sys=47 FOLDER='/opt/idssys/powerwall' LOGFOLDER=${FOLDER}/logs LOGARCHIVE=${FOLDER}/logs/archive +TMPFOLDER=${FOLDER}/.tmp [ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER} [ ! -d ${LOGARCHIVE} ] && mkdir ${LOGARCHIVE} +[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER} logfile=${LOGFOLDER}/logfile logpower=${LOGFOLDER}/log-power diff --git a/powerwall.sh b/powerwall.sh index bdd673e4..4886365b 100755 --- a/powerwall.sh +++ b/powerwall.sh @@ -129,9 +129,8 @@ Shutting down servers!!" 1 echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} echo -e "($datetime) - ${SENSOR} - ${temp_f}'F - CRITICAL TEMP - (Difference of ${temp_diff}')" >> ${logfile} last_temp[${SENSOR}]=$temp_f - SHUTDOWN_MAIN ${SENSOR} - SENDNOTICE "SERVERS SHUTDOWN" "($datetime) Main servers have been shutdown" 1 - echo -e "($datetime) - ${temp_f}'F - Main servers have been shutdown" >> ${logtemp} + SHUTDOWN_MAIN ${SENSOR} & + echo -e "($datetime) - ${temp_f}'F - Shutting down MAIN servers" >> ${logtemp} fi elif [ $(bc -l <<< "$temp_f >= $temp_crit_sys") -eq 1 ]; then @@ -142,10 +141,9 @@ Shutting down system servers!!" 1 echo "($datetime) - ${temp_f}'F - alert sent" >> ${logtemp} echo -e "($datetime) - ${temp_f}'F - ${SENSOR} - CRITICAL TEMP SYSTEM - (Difference of ${temp_diff}')" >> ${logfile} last_temp[${SENSOR}]=$temp_f - SHUTDOWN_SYS ${SENSOR} TEMP - SHUTDOWN_CRIT ${SENSOR} - SENDNOTICE "SERVERS SHUTDOWN" "($datetime) System servers have been shutdown" 1 - echo -e "($datetime) - ${temp_f}'F - System servers have been shutdown" >> ${logtemp} + SHUTDOWN_SYS ${SENSOR} TEMP & + SHUTDOWN_CRIT ${SENSOR} & + echo -e "($datetime) - ${temp_f}'F - Shutting down SYS servers" >> ${logtemp} fi else if [ ${last_temp[${SENSOR}]} -gt 1 ]; then @@ -169,12 +167,17 @@ NORMAL TEMP: ${temp_f}'F" fi last_temp[${SENSOR}]=1 fi - if [ "${temp_h}" != "" ]; then - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" - else - QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" + + [ -f ${TMPFOLDER}/${SENSOR}.insert ] && errtime=$(expr `date +%s` - $(stat -c %Y ${TMPFOLDER}/${SENSOR}.insert)) || errtime=0 + if [ ${errtime} -ge 300 ]; then + if [ "${temp_h}" != "" ]; then + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`, \`hum\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}','${temp_h}')" + else + QRY="USE servermonitor; INSERT INTO sensor_data (\`sensorid\`, \`date\`, \`temp\`) VALUES ('${SENSOR_ID[${SENSOR}]}','${datetime}','${temp_f}')" + fi + ${mysql_conn} -e "${QRY}" + touch ${TMPFOLDER}/${SENSOR}.insert fi - ${mysql_conn} -e "${QRY}" else SENDNOTICE "${SENSOR} ERROR" "($datetime) ERROR reading temperature" 1 echo "($datetime) - alert sent" >> ${logtemp} @@ -239,6 +242,7 @@ CHECKPOWER_SERVICE(){ fi fi voltstatus=1 + rm -f ${TMPFOLDER}/power.ac.* else echo "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logpower} echo -e "($datetime) - ${INPUTACV}V - POWER OFF/LOW POWER" >> ${logfile} @@ -250,29 +254,33 @@ CHECKPOWER_SERVICE(){ last_battvolt=$BATTVOLT fi voltstatus=3 + touch ${TMPFOLDER}/power.ac.low fi if [ ${BATTVOLT%.*} -lt ${min_battvolt} ] && [ ${BATTVOLT%.*} -gt ${min_battvolt_sys} ]; then echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower} echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile} + touch ${TMPFOLDER}/power.dc.low if [ $battstatus -lt 2 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down main servers..." >> ${logpower} - SENDNOTICE "BATT-VOLT LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V + SENDNOTICE "BATT-VOLT LOW: SHUTDOWN MAIN SERVERS" "($datetime) Battery Voltage LOW: ${BATTVOLT}V Shutting down main servers" 1 - SHUTDOWN_MAIN server + SHUTDOWN_MAIN power & battstatus=2 fi elif [ ${BATTVOLT%.*} -le ${min_battvolt_sys} ]; then echo "($datetime) - ${BATTVOLT}V - LOW BATTERY Voltage" >> ${logpower} echo -e "($datetime) - ${BATTVOLT}V - LOW BATTERY VOLTAGE" >> ${logfile} + touch ${TMPFOLDER}/power.dc.low if [ $battstatus -lt 3 ] && [ ${INPUTACV} -lt ${min_acvolt} ]; then echo "($datetime) - ${BATTVOLT}V - Shutting down all remaining servers..." >> ${logpower} - SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V + SENDNOTICE "BATT-VOLT REALLY LOW: SHUTDOWN SYS SERVERS" "($datetime) Battery Voltage REALLY LOW: ${BATTVOLT}V Shutting down all servers" 1 - SHUTDOWN_SYS server + SHUTDOWN_SYS power & battstatus=3 + fi else @@ -281,7 +289,7 @@ CHECKPOWER_SERVICE(){ if [ $battstatus -eq 0 ]; then echo "($datetime) - Battery: ${BATTVOLT}V - Service Startup - Normal voltage detected" >> ${logpower} fi - + rm -f ${TMPFOLDER}/power.dc.* battstatus=1 fi @@ -320,9 +328,14 @@ POWERLOGGER_SERVICE(){ ${mysql_conn} -e "USE servermonitor; ${QRY}" QRY="INSERT INTO inv_jsonstats (\`date\`, \`data\`) VALUES ('${datetime}','${JSONSTATS}')" ${mysql_conn} -e "USE servermonitor; ${QRY}" - - sleep 1m + + if [ "$(compgen -G "${TMPFOLDER}/power.ac.*")" != "" ] || [ "$(compgen -G "${TMPFOLDER}/power.dc.*")" != "" ]; then + sleep 1m + else + sleep 5m + fi + done } @@ -381,12 +394,38 @@ CHECK_SERVICES(){ } SHUTDOWN_MAIN(){ - SHUTDOWN_SERVERS MAIN + if [ "${1}" == "ServerRoomTH"]; then + SHUTDOWN_SERVERS MAIN ${2} + reason='Server Room Overheated!' + sendnotice=true + + else + sendnotice=false + fi + + if [ "${sendnotice}" == "true"]; then + datetime=`date +'%Y-%m-%d %H:%M:%S'` + SENDNOTICE "MAIN SERVERS SHUTDOWN" "($datetime) MAIN servers have been shutdown + ${reason}" 1 + echo -e "($datetime) - Main servers have been shutdown, ${reason}" >> ${logfile} + fi + echo } SHUTDOWN_SYS(){ - SHUTDOWN_SERVERS SYS - curl --data "cmd=PWRoff" http://10.10.0.61/cmd + if [ "${1}" == "ServerRoomTH"]; then + SHUTDOWN_SERVERS SYS ${2} + reason='Server Room Overheated!' + fi + + if [ "${sendnotice}" == "true"]; then + datetime=`date +'%Y-%m-%d %H:%M:%S'` + SENDNOTICE "SYS SERVERS SHUTDOWN" "($datetime) SYS servers have been shutdown + ${reason}" 1 + echo -e "($datetime) - SYS servers have been shutdown, ${reason}" >> ${logfile} + fi + + # curl --data "cmd=PWRoff" http://10.10.0.61/cmd echo } SHUTDOWN_CRIT(){ @@ -592,7 +631,8 @@ fi ;; checktemp) CHECKTEMP ${2};; checkpower) CHECKPOWER;; - test) LOGROTATE ${2};; + test) TEST ${2};; + logrotate) LOGROTATE;; iscsi-vm-shutdown) ISCSIVMSHUTDOWN;; shutdownhost) SHUTDOWN_SERVER ${2};;