#!/usr/bin/env bash VERS='5.1.30-12172023' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck ' CERT_DAEMON='/snap/bin/certbot' declare -i errtime [ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt NM_SRVCOPT=(start stop restart reload enable disable) if [ -f ${NM_FOLDER}/conf/defaults.local.inc ]; then declare -A NM_HOSTS declare -A NM_SERVICES_CHECK declare -A NM_DOCKERS_CHECK declare -A NM_REPL_CHECK declare -A NM_REPL_NGINX_PATHS declare -A NM_HOSTNAMES declare -A NM_NODETYPES declare -A NM_SERVICE_DESC declare -A NM_DOCKER_DESC declare -A NM_DOCKER_COMPOSE_LOC declare -A NM_REPL_CHECK_LOC declare -A NM_REPL_DESC declare -A NM_SINGLESRVR_SERVICES declare -A NM_SINGLESRVR_DOCKERS declare -A NM_SINGLESRVR_IP declare -A NM_CERTPATHS declare -A NM_WWWPATHS declare -A NM_DYNDNS_SITES source ${NM_FOLDER}/conf/defaults.local.inc IFS=, LOCAL_SERVICES=(${LOCAL_SERVICES}) unset IFS NODETYPES=$(for k in "${!NM_NODETYPES[@]}"; do echo "${NM_NODETYPES["$k"]} ${k}"; done | sort -f | while read desc nmtype; do echo ${nmtype}; done) IFS=$'\n' NM_NODE_TYPES=(${NODETYPES}); unset IFS # printf "[%s]\n" "${NM_NODE_TYPES[@]}" for ntype in "${NM_NODE_TYPES[@]}"; do arr ${ntype}_HOSTS arr ${ntype}_SERVICES_CHECK arr ${ntype}_DOCKERS_CHECK arr ${ntype}_REPL_CHECK arr ${ntype}_SINGLESRVR_SERVICES arr ${ntype}_SINGLESRVR_DOCKERS arr ${ntype}_SINGLESRVR_IP IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_HOSTS $value done if [ "${NM_SERVICES_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SERVICES_CHECK $value done fi if [ "${NM_DOCKERS_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_DOCKERS_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_DOCKERS_CHECK $value done fi if [ "${NM_REPL_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_REPL_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_REPL_CHECK $value done fi if [ "${NM_SINGLESRVR_SERVICES[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_SERVICES[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_SERVICES $value done fi if [ "${NM_SINGLESRVR_DOCKERS[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_DOCKERS[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_DOCKERS $value done fi if [ "${NM_SINGLESRVR_IP[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_IP[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_IP $value done # else # declare "${ntype}_SINGLESRVR_IP"="" fi done fi [ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt [ "${NM_BACKUP_FOLDER}" == "" ] && NM_BACKUP_FOLDER=/opt/idssys/backups NM_SCRIPT=${NM_FOLDER}/nodemgmt-scripts.sh NM_TMPFOLDER=${NM_FOLDER}/.tmp [ "${NM_LOGFOLDER}" == "" ] && NM_LOGFOLDER=${NM_FOLDER}/logs NM_LOGFILE=${NM_LOGFOLDER}/logfile [ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER} [ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER} [ "${NM_RENOTIFY_TIMEOUT}" == "" ] && NM_RENOTIFY_TIMEOUT=3600 NM_PDNS_WANDNS='wdns.scity.us' NM_PDNS_LANDNS='ldns.scity.us' TSI=$(/sbin/ip link | grep tailscale0) && [ ${#TSI} != 0 ] && RUN_NODE_TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1) || RUN_NODE_TSIP= [ "$(ip route | awk '/default/ { print $5 }')" == "link" ] && RNIP=$(ip addr show $(ip route | awk '/1 dev eth0/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) || RNIP=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) if [ "${RNIP}" == "${NM_NODEMANAGER}" ]; then RUN_NODE_TYPE=NM; RUN_NODE_IP=${RNIP} else for NTYPE in "${NM_NODE_TYPES[@]}"; do var=${NTYPE}_HOSTS[@] RUN_NODE_TYPE="" for nip in "${!var}"; do # echo "${NTYPE} - $nip - ${RNIP} - ${RUN_NODE_TSIP}" if [[ "${RNIP}" == *"${nip}"* ]]; then RUN_NODE_TYPE=${NTYPE}; RUN_NODE_IP=${RNIP} break 2 elif [[ "${RUN_NODE_TSIP}" == *"${nip}"* ]]; then RUN_NODE_TYPE=${NTYPE}; RUN_NODE_IP=${RNIP} break 2 fi done done fi # echo "HERE: RUN_NODE_TYPE=$RUN_NODE_TYPE RUN_NODE_IP=$RUN_NODE_IP RUN_NODE_TSIP=$RUN_NODE_TSIP (RNIP=$RNIP TSI=$TSI)" [ "${NM_REPL_NGINX_PATHS}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_NGINXPATH=${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]} || NM_NGINXPATH=/etc/nginx if [ "${NM_CERTPATHS}" != "" ]; then if [ "${NM_CERTPATHS[${RUN_NODE_TYPE}]}" != "" ]; then NM_CERTPATH=${NM_CERTPATHS[${RUN_NODE_TYPE}]} NM_CERTNODE=${RUN_NODE_TYPE} else NM_CERTPATH=${NM_CERTPATHS[@]} NM_CERTNODE=${!NM_CERTPATHS[@]} fi else NM_CERTPATH=/etc/letsencrypt fi [ "${NM_WWWPATHS}" != "" ] && [ "${NM_WWWPATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_WWWPATH=${NM_WWWPATHS[${RUN_NODE_TYPE}]} || NM_WWWPATH=/var/www declare -A NM_SRVCOPTS NM_SRVCOPTS['status']='Status' NM_SRVCOPTS['start']='Start' NM_SRVCOPTS['stop']='Stopp' NM_SRVCOPTS['restart']='Restart' NM_SRVCOPTS['reload']='Reload' NM_SRVCOPTS['enable']='Enabl' NM_SRVCOPTS['disable']='Disabl' NM_SRVCOPTS['daemon-reload']='Daemon-Reload' declare -A NM_BACKUP_ITEMS NM_BACKUP_ITEMS['nginx-settings']=${NM_NGINXPATH} NM_BACKUP_ITEMS['nginx-logs']=/var/log/nginx NM_BACKUP_ITEMS['letsencrypt-certs']=${NM_CERTPATH} NM_BACKUP_ITEMS['webserver-files']=${NM_WWWPATH} NM_BACKUP_ITEMS['haproxy']=/etc/haproxy NM_BACKUP_ITEMS['keepalived']=/etc/keepalived NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban NM_BACKUP_ITEMS['nmg-conf']=${NM_FOLDER}/conf # NM_BACKUP_ITEMS[nginx]=/ for NTYPE in "${NM_NODE_TYPES[@]}"; do if [[ "${NM_DOCKERS_CHECK[${NTYPE}]}" == *"authelia"* ]]; then NM_AUTHELIA_IP=${NM_SINGLESRVR_IP[${NTYPE}]} break fi done GET-CHECKCERT-DOMAINS(){ declare -A CHECKCERT_DOMAINS IFS=$'\n' for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/conf/ssl-domain-checks.conf`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " CHECKCERT_DOMAINS[${HOST}]=${PORT} done unset IFS } DISP_HEADER(){ declare -i cw; declare -i spc1; declare -i c if [ "$1" = true ]; then clear fi echo echo -e "${idsCL[LightGreen]} NodeMgmt - Node Monitoring & Management${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}" DIVIDER . lightGreen echo } SENDNOTICE(){ [ "${PUSHOVER_USER_TOKEN}" != "" ] && PUSH_TO_MOBILE "${2} $(date)" "${1}" ${3} & [ "${EMAIL_NOTICE}" != "" ] && echo -e "${2}\n\n$(date)" | mail -s "${1}" ${EMAIL_NOTICE} }