228 lines
7.0 KiB
Bash
Executable File
228 lines
7.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
VERS='5.1.89-02192024'
|
|
NM_BETA=false
|
|
|
|
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck '
|
|
CERT_DAEMON='/snap/bin/certbot'
|
|
|
|
declare -i errtime
|
|
[ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt
|
|
NM_SRVCOPT=(start stop restart reload enable disable)
|
|
|
|
if [ -f ${NM_FOLDER}/conf/defaults.local.inc ]; then
|
|
declare -A NM_HOSTS
|
|
declare -A NM_SERVICES_CHECK
|
|
declare -A NM_DOCKERS_CHECK
|
|
declare -A NM_REPL_CHECK
|
|
declare -A NM_REPL_NGINX_PATHS
|
|
declare -A NM_HOSTNAMES
|
|
declare -A NM_NODETYPES
|
|
declare -A NM_SERVICE_DESC
|
|
declare -A NM_DOCKER_DESC
|
|
declare -A NM_DOCKER_COMPOSE_LOC
|
|
declare -A NM_REPL_CHECK_LOC
|
|
declare -A NM_REPL_DESC
|
|
declare -A NM_SINGLESRVR_SERVICES
|
|
declare -A NM_SINGLESRVR_DOCKERS
|
|
declare -A NM_SINGLESRVR_IP
|
|
declare -A NM_CERTPATHS
|
|
declare -A NM_WWWPATHS
|
|
declare -A NM_DYNDNS_SITES
|
|
declare -A NM_HOST_CPULEVELS
|
|
|
|
source ${NM_FOLDER}/conf/defaults.local.inc
|
|
|
|
[ "${NM_REPL_CHECK_TIMEOUT}" != "" ] && NM_REPL_CHECK_TIMEOUT=${NM_REPL_CHECK_TIMEOUT} || NM_REPL_CHECK_TIMEOUT=5 #MINUTES
|
|
|
|
IFS=,
|
|
LOCAL_SERVICES=(${LOCAL_SERVICES})
|
|
unset IFS
|
|
|
|
NODETYPES=$(for k in "${!NM_NODETYPES[@]}"; do echo "${NM_NODETYPES["$k"]} ${k}"; done | sort -f | while read desc nmtype; do echo ${nmtype}; done)
|
|
IFS=$'\n' NM_NODE_TYPES=(${NODETYPES}); unset IFS
|
|
# printf "[%s]\n" "${NM_NODE_TYPES[@]}"
|
|
|
|
for ntype in "${NM_NODE_TYPES[@]}"; do
|
|
arr ${ntype}_HOSTS
|
|
arr ${ntype}_SERVICES_CHECK
|
|
arr ${ntype}_DOCKERS_CHECK
|
|
arr ${ntype}_REPL_CHECK
|
|
arr ${ntype}_SINGLESRVR_SERVICES
|
|
arr ${ntype}_SINGLESRVR_DOCKERS
|
|
arr ${ntype}_SINGLESRVR_IP
|
|
|
|
IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_HOSTS $value
|
|
done
|
|
|
|
if [ "${NM_SERVICES_CHECK[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_SERVICES_CHECK $value
|
|
done
|
|
fi
|
|
|
|
if [ "${NM_DOCKERS_CHECK[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_DOCKERS_CHECK[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_DOCKERS_CHECK $value
|
|
done
|
|
fi
|
|
|
|
if [ "${NM_REPL_CHECK[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_REPL_CHECK[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_REPL_CHECK $value
|
|
done
|
|
fi
|
|
|
|
if [ "${NM_SINGLESRVR_SERVICES[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_SINGLESRVR_SERVICES[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_SINGLESRVR_SERVICES $value
|
|
done
|
|
fi
|
|
|
|
if [ "${NM_SINGLESRVR_DOCKERS[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_SINGLESRVR_DOCKERS[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_SINGLESRVR_DOCKERS $value
|
|
done
|
|
fi
|
|
|
|
if [ "${NM_SINGLESRVR_IP[${ntype}]}" != "" ]; then
|
|
IFS=,; var=(${NM_SINGLESRVR_IP[${ntype}]}); unset IFS
|
|
for value in "${var[@]}"; do
|
|
arr_insert ${ntype}_SINGLESRVR_IP $value
|
|
done
|
|
# else
|
|
# declare "${ntype}_SINGLESRVR_IP"=""
|
|
fi
|
|
done
|
|
|
|
fi
|
|
|
|
[ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt
|
|
[ "${NM_BACKUP_FOLDER}" == "" ] && NM_BACKUP_FOLDER=/opt/idssys/backups
|
|
NM_SCRIPT=${NM_FOLDER}/nodemgmt-scripts.sh
|
|
NM_TMPFOLDER=${NM_FOLDER}/.tmp
|
|
[ "${NM_LOGFOLDER}" == "" ] && NM_LOGFOLDER=${NM_FOLDER}/logs
|
|
NM_LOGFILE=${NM_LOGFOLDER}/logfile
|
|
|
|
[ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER}
|
|
[ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER}
|
|
|
|
[ "${NM_RENOTIFY_TIMEOUT}" == "" ] && NM_RENOTIFY_TIMEOUT=3600
|
|
|
|
NM_PDNS_WANDNS='wdns.scity.us'
|
|
NM_PDNS_LANDNS='ldns.scity.us'
|
|
|
|
TSI=$(/sbin/ip link | grep tailscale0) && [ ${#TSI} != 0 ] && RUN_NODE_TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1) || RUN_NODE_TSIP=
|
|
[ "$(ip route | awk '/default/ { print $5 }')" == "link" ] && RNIP=$(ip addr show $(ip route | awk '/1 dev eth0/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) ||
|
|
RNIP=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1)
|
|
if [ "${RNIP}" == "${NM_NODEMANAGER}" ]; then
|
|
RUN_NODE_TYPE=NM;
|
|
RUN_NODE_IP=${RNIP}
|
|
else
|
|
for NTYPE in "${NM_NODE_TYPES[@]}"; do
|
|
var=${NTYPE}_HOSTS[@]
|
|
RUN_NODE_TYPE=""
|
|
for nip in "${!var}"; do
|
|
# echo "${NTYPE} - $nip - ${RNIP} - ${RUN_NODE_TSIP}"
|
|
if [[ "${RNIP}" == *"${nip}"* ]]; then
|
|
RUN_NODE_TYPE=${NTYPE};
|
|
RUN_NODE_IP=${RNIP}
|
|
break 2
|
|
elif [[ "${RUN_NODE_TSIP}" == *"${nip}"* ]]; then
|
|
RUN_NODE_TYPE=${NTYPE};
|
|
RUN_NODE_IP=${RNIP}
|
|
break 2
|
|
fi
|
|
done
|
|
done
|
|
fi
|
|
# echo "HERE: RUN_NODE_TYPE=$RUN_NODE_TYPE RUN_NODE_IP=$RUN_NODE_IP RUN_NODE_TSIP=$RUN_NODE_TSIP (RNIP=$RNIP TSI=$TSI)"
|
|
[ "${NM_REPL_NGINX_PATHS}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_NGINXPATH=${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]} || NM_NGINXPATH=/etc/nginx
|
|
if [ "${NM_CERTPATHS}" != "" ]; then
|
|
if [ "${NM_CERTPATHS[${RUN_NODE_TYPE}]}" != "" ]; then
|
|
NM_CERTPATH=${NM_CERTPATHS[${RUN_NODE_TYPE}]}
|
|
NM_CERTNODE=${RUN_NODE_TYPE}
|
|
else
|
|
NM_CERTPATH=${NM_CERTPATHS[@]}
|
|
NM_CERTNODE=${!NM_CERTPATHS[@]}
|
|
fi
|
|
else
|
|
NM_CERTPATH=/etc/letsencrypt
|
|
fi
|
|
[ "${NM_WWWPATHS}" != "" ] && [ "${NM_WWWPATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_WWWPATH=${NM_WWWPATHS[${RUN_NODE_TYPE}]} || NM_WWWPATH=/var/www
|
|
|
|
declare -A NM_SRVCOPTS
|
|
NM_SRVCOPTS['status']='Status'
|
|
NM_SRVCOPTS['start']='Start'
|
|
NM_SRVCOPTS['stop']='Stopp'
|
|
NM_SRVCOPTS['restart']='Restart'
|
|
NM_SRVCOPTS['reload']='Reload'
|
|
NM_SRVCOPTS['enable']='Enabl'
|
|
NM_SRVCOPTS['disable']='Disabl'
|
|
NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
|
|
|
|
declare -A NM_BACKUP_ITEMS
|
|
NM_BACKUP_ITEMS['nginx-settings']=${NM_NGINXPATH}
|
|
NM_BACKUP_ITEMS['nginx-logs']=/var/log/nginx
|
|
NM_BACKUP_ITEMS['letsencrypt-certs']=${NM_CERTPATH}
|
|
NM_BACKUP_ITEMS['webserver-files']=${NM_WWWPATH}
|
|
NM_BACKUP_ITEMS['haproxy']=/etc/haproxy
|
|
NM_BACKUP_ITEMS['keepalived']=/etc/keepalived
|
|
NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban
|
|
NM_BACKUP_ITEMS['nmg-conf']=${NM_FOLDER}/conf
|
|
# NM_BACKUP_ITEMS[nginx]=/
|
|
|
|
for NTYPE in "${NM_NODE_TYPES[@]}"; do
|
|
if [[ "${NM_DOCKERS_CHECK[${NTYPE}]}" == *"authelia"* ]]; then
|
|
NM_AUTHELIA_IP=${NM_SINGLESRVR_IP[${NTYPE}]}
|
|
break
|
|
fi
|
|
done
|
|
|
|
GET-CHECKCERT-DOMAINS(){
|
|
declare -A CHECKCERT_DOMAINS
|
|
IFS=$'\n'
|
|
for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/conf/ssl-domain-checks.conf`; do
|
|
HOST=${LINE%% *}
|
|
PORT=${LINE#* }
|
|
IFS=" "
|
|
CHECKCERT_DOMAINS[${HOST}]=${PORT}
|
|
done
|
|
unset IFS
|
|
}
|
|
|
|
DISP_HEADER(){
|
|
declare -i cw; declare -i spc1; declare -i c
|
|
if [ "$1" = true ]; then
|
|
clear
|
|
fi
|
|
echo
|
|
echo -e "${idsCL[LightGreen]} NodeMgmt - Node Monitoring & Management${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}"
|
|
DIVIDER . lightGreen
|
|
echo
|
|
}
|
|
|
|
SENDNOTICE(){
|
|
if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then
|
|
SN_TIT="$(echo -e "${1}" | sed "s/\%/\%25 /g")"
|
|
SN_MSG="$(echo -e "${2}" | sed "s/\%/\%25 /g")"
|
|
PUSH_TO_MOBILE "${SN_MSG}
|
|
|
|
$(date)" "${SN_TIT}" ${3} &
|
|
fi
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo -e "${2}\n\n$(date)" | mail -s "${1}" ${EMAIL_NOTICE}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|