#!/usr/bin/env bash VERS='5.6.52-11192025' NM_BETA=false noheader=' test service status-check nightlyrephp7.3-fpm,new backup report reportrun check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' declare -i errtime [ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt NM_SRVCOPT=(start stop restart reload enable disable) # TSI=$(/sbin/ip link | grep tailscale0) # [ ${#TSI} != 0 ] && RUN_NODE_TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1) || RUN_NODE_TSIP="" if [ "$(ip route | awk '/default/ { print $5 }')" == "link" ]; then RNIP=$(ip addr show $(ip route | awk '/1 dev eth0/ { print $5; exit }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) elif [ "$(ip a | awk '/wg0/')" ]; then RNIP=$(ip a | awk '/wg0/' | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) else RNIP=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) fi if [ -f ${NM_FOLDER}/conf/defaults.local.inc ]; then declare -A NM_HOSTS declare -A NM_SERVICES_CHECK declare -A NM_DOCKERS_CHECK declare -A NM_REPL_CHECK declare -A NM_REPL_NGINX_PATHS declare -A NM_HOSTNAMES declare -A NM_NODETYPES declare -A NM_SERVICE_DESC declare -A NM_DOCKER_DESC declare -A NM_DOCKER_COMPOSE_LOC declare -A NM_REPL_CHECK_LOC declare -A NM_REPL_DESC declare -A NM_SINGLESRVR_SERVICES declare -A NM_SINGLESRVR_DOCKERS declare -A NM_SINGLESRVR_IP declare -A NM_CERTPATHS declare -A NM_WWWPATHS declare -A NM_DYNDNS_SITES declare -A NM_HOST_CPULEVELS source ${NM_FOLDER}/conf/defaults.local.inc [ "${NM_REPL_CHECK_TIMEOUT}" != "" ] && NM_REPL_CHECK_TIMEOUT=${NM_REPL_CHECK_TIMEOUT} || NM_REPL_CHECK_TIMEOUT=5 #MINUTES [ "${CERT_DAEMON}" == "" ] && CERT_DAEMON='/opt/certbot/bin/certbot' IFS=, LOCAL_SERVICES=(${LOCAL_SERVICES}) unset IFS NODETYPES=$(for k in "${!NM_NODETYPES[@]}"; do echo "${NM_NODETYPES["$k"]} ${k}"; done | sort -f | while read desc nmtype; do echo ${nmtype}; done) IFS=$'\n' NM_NODE_TYPES=(${NODETYPES}); unset IFS # printf "[%s]\n" "${NM_NODE_TYPES[@]}" for ntype in "${NM_NODE_TYPES[@]}"; do arr ${ntype}_HOSTS arr ${ntype}_SERVICES_CHECK arr ${ntype}_DOCKERS_CHECK arr ${ntype}_REPL_CHECK arr ${ntype}_SINGLESRVR_SERVICES arr ${ntype}_SINGLESRVR_DOCKERS arr ${ntype}_SINGLESRVR_IP IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_HOSTS $value done if [ "${NM_SERVICES_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SERVICES_CHECK $value done fi if [ "${NM_DOCKERS_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_DOCKERS_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_DOCKERS_CHECK $value done arr_insert ${ntype}_DOCKERS_CHECK watchtower ([[ "${RNIP}" != "10.6."* ]] && [ "${ntype}" != "ESPH" ]) && arr_insert ${ntype}_DOCKERS_CHECK portainer_agent fi if [ "${NM_REPL_CHECK[${ntype}]}" != "" ]; then IFS=,; var=(${NM_REPL_CHECK[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_REPL_CHECK $value done fi if [ "${NM_SINGLESRVR_SERVICES[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_SERVICES[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_SERVICES $value done fi if [ "${NM_SINGLESRVR_DOCKERS[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_DOCKERS[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_DOCKERS $value done fi if [ "${NM_SINGLESRVR_IP[${ntype}]}" != "" ]; then IFS=,; var=(${NM_SINGLESRVR_IP[${ntype}]}); unset IFS for value in "${var[@]}"; do arr_insert ${ntype}_SINGLESRVR_IP $value done # else # declare "${ntype}_SINGLESRVR_IP"="" fi done fi [ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt [ "${NM_BACKUP_FOLDER}" == "" ] && NM_BACKUP_FOLDER=/opt/idssys/backups NM_SCRIPT=${NM_FOLDER}/nodemgmt-scripts.sh NM_TMPFOLDER=${NM_FOLDER}/.tmp PW_TMPFOLDER=${NM_TMPFOLDER}/powerwall [ "${NM_LOGFOLDER}" == "" ] && NM_LOGFOLDER=${NM_FOLDER}/logs NM_LOGFILE=${NM_LOGFOLDER}/logfile [ ! -d ${NM_TMPFOLDER} ] && mkdir ${NM_TMPFOLDER} [ ! -d ${NM_LOGFOLDER} ] && mkdir ${NM_LOGFOLDER} [ "${NM_RENOTIFY_TIMEOUT}" == "" ] && NM_RENOTIFY_TIMEOUT=3600 [ "${NM_ENABLE_RENOTIFY}" == "" ] && NM_ENABLE_RENOTIFY=1 NM_PDNS_WANDNS='wdns.scity.us' NM_PDNS_LANDNS='ldns.scity.us' if [ "${RNIP}" == "${NM_NODEMANAGER}" ]; then IS_NODEMANAGER=1 # RUN_NODE_TYPE=NM; # RUN_NODE_IP=${RNIP} else IS_NODEMANAGER=0 fi for NTYPE in "${NM_NODE_TYPES[@]}"; do # if [ "${1}" != "service" ] || ([ "${1}" == "service" ] && ([[ "${NM_SERVICES_CHECK[${NTYPE}]}" = *"${2}"* ]] || [ "${NTYPE}" == "${2^^}" ])); then var=${NTYPE}_HOSTS[@] RUN_NODE_TYPE="" for nip in "${!var}"; do # echo "${NTYPE} - $nip - ${RNIP} - ${RUN_NODE_TSIP}" if [[ "${RNIP}" == *"${nip}"* ]]; then RUN_NODE_TYPE=${NTYPE}; RUN_NODE_IP=${RNIP} break 2 # elif [[ "${RUN_NODE_TSIP}" == *"${nip}"* ]]; then # RUN_NODE_TYPE=${NTYPE}; # RUN_NODE_IP=${RNIP} # break 2 fi done # fi done # fi # echo "HERE1: RUN_NODE_TYPE=$RUN_NODE_TYPE RUN_NODE_IP=$RUN_NODE_IP RUN_NODE_TSIP=$RUN_NODE_TSIP (RNIP=$RNIP TSI=$TSI)" [ "${NM_REPL_NGINX_PATHS}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_NGINXPATH=${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]} || NM_NGINXPATH=/etc/nginx if [ "${NM_CERTPATHS}" != "" ]; then if [ "${NM_CERTPATHS[${RUN_NODE_TYPE}]}" != "" ]; then NM_CERTPATH=${NM_CERTPATHS[${RUN_NODE_TYPE}]} NM_CERTNODE=${RUN_NODE_TYPE} else NM_CERTPATH=${NM_CERTPATHS[@]} NM_CERTNODE=${!NM_CERTPATHS[@]} fi else NM_CERTPATH=/etc/letsencrypt fi [ "${NM_WWWPATHS}" != "" ] && [ "${NM_WWWPATHS[${RUN_NODE_TYPE}]}" != "" ] && NM_WWWPATH=${NM_WWWPATHS[${RUN_NODE_TYPE}]} || NM_WWWPATH=/var/www declare -A NM_SRVCOPTS NM_SRVCOPTS['status']='Status' NM_SRVCOPTS['start']='Start' NM_SRVCOPTS['stop']='Stopp' NM_SRVCOPTS['restart']='Restart' NM_SRVCOPTS['reload']='Reload' NM_SRVCOPTS['enable']='Enabl' NM_SRVCOPTS['disable']='Disabl' NM_SRVCOPTS['daemon-reload']='Daemon-Reload' declare -A NM_BACKUP_ITEMS NM_BACKUP_ITEMS['nginx-settings']=${NM_NGINXPATH} NM_BACKUP_ITEMS['nginx-logs']=/var/log/nginx NM_BACKUP_ITEMS['letsencrypt-certs']=${NM_CERTPATH} NM_BACKUP_ITEMS['webserver-files']=${NM_WWWPATH} NM_BACKUP_ITEMS['haproxy']=/etc/haproxy NM_BACKUP_ITEMS['keepalived']=/etc/keepalived NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban NM_BACKUP_ITEMS['nmg-conf']=${NM_FOLDER}/conf # NM_BACKUP_ITEMS[nginx]=/ NM_MEDIA_SERVICES="radarr,radarr3d,sonarr,prowlarr,qbittorrent,bazarr,snap.tautulli.tautulli,snap.overseerr.daemon,jackett,lidarr,ombi" for NTYPE in "${NM_NODE_TYPES[@]}"; do if [[ "${NM_DOCKERS_CHECK[${NTYPE}]}" == *"authelia"* ]]; then NM_AUTHELIA_IP=${NM_SINGLESRVR_IP[${NTYPE}]} break fi done GET-CHECKCERT-DOMAINS(){ declare -A CHECKCERT_DOMAINS IFS=$'\n' for LINE in `egrep -v '(^#|^$)' ${NM_FOLDER}/conf/ssl-domain-checks.conf`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " CHECKCERT_DOMAINS[${HOST}]=${PORT} done unset IFS } DISP_HEADER(){ declare -i cw; declare -i spc1; declare -i c if [ "$1" = true ]; then clear fi echo echo -e "${idsCL[LightGreen]} NodeMgmt - Node Monitoring & Management${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}" DIVIDER . lightGreen echo } CHECK_HOST_QC(){ if [ ! -z ${1+x} ]; then ping -qw 2 -c 1 ${1} > /dev/null 2>&1 [ $? -ne 0 ] && echo false || echo true else echo false fi } SENDNOTICE(){ if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then SN_TIT="$(echo -e "${1}" | sed "s/\%/\%25 /g")" SN_MSG="$(echo -e "${2}" | sed "s/\%/\%25 /g")" PUSH_TO_MOBILE "${SN_MSG} [${NM_HOSTNAMES[${RNIP}]}] $(date)" "${SN_TIT}" ${3} & fi [ "${EMAIL_NOTICE}" != "" ] && echo -e "${2}\n\n$(date)" | mail -s "${1}" ${EMAIL_NOTICE} }