diff --git a/defaults.local.example.inc b/defaults.local.example.inc new file mode 100755 index 00000000..eccdf7df --- /dev/null +++ b/defaults.local.example.inc @@ -0,0 +1,346 @@ +#!/usr/bin/env bash + +VERS='4.7.153-07022023' + +# NODETYPES=WEB +NODETYPES=LB,LPD,MM,MYSQL,NC,PW,WPD,WM,WEB +MYSQLHOSTS=10.10.1.51,10.10.1.52 +WEBHOSTS=10.10.1.121,10.10.1.122 +LBHOSTS=10.10.10.81,10.10.10.82 +NCHOSTS=10.10.10.42 +WPDHOSTS=100.100.55.2,100.100.55.3,10.10.1.28 +LPDHOSTS=10.10.10.10,10.10.10.11,10.2.1.2 +MMHOSTS=10.10.1.24 +PRIMARYHOST=10.10.1.51 +MYSQLMANAGER=10.10.10.50 +GLHOSTS=10.10.1.41,10.10.1.42,10.10.1.121,10.10.1.122 +PWHOSTS=10.10.0.240 +WMHOSTS=100.100.55.5 + +NODESERVICES="mariadb,mysql,nginx,gitea,pdnsadmin,pdnsadmin.socket,haproxy,keepalived,maxscale,postgresql,web,php5.6-fpm,php7.2-fpm,php7.3-fpm" +NODESERVICES="${NODESERVICES},php7.4-fpm,syncthing,pdns,proxysql,radarr,sonarr,jackett,qbittorrent,snap.tautulli.tautulli,ombi,glusterd" +NODESERVICES="${NODESERVICES},redis-server,mosquitto,tailscaled,cmon,monitor-power,monitor-temp" + +MYSQLSERVICES_CHECK=mariadb,proxysql,keepalived +WEBSERVICES_CHECK=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived +LBSERVICES_CHECK=nginx,haproxy,keepalived +NCSERVICES_CHECK=nginx,php8.1-fpm,redis-server +WPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,tailscaled +LPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,keepalived +MMSERVICES_CHECK=bazarr,radarr,sonarr,prowlarr,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm +PWSERVICES_CHECK=mosquitto,monitor-power,monitor-temp +WMSERVICES_CHECK=nginx,tailscaled + +WEBDOCKER=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia +WMDOCKER=uptime-kuma +PWDOCKER=uptime-kuma +#LBDOCKER=nginx-proxy_manager + +WEB_SINGLESRVRSERVICES=gitea +WEB_SINGLESRVRDOCKERS=vaultwarden,heimdall,authelia +WEB_SINGLESRVR_IP=10.10.1.20 + +LB_NGINX_SITES='/opt/lb-data/nginx/sites-enabled' +WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled' + +noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns ' +CERT_DAEMON='/snap/bin/certbot' +FOLDER=/opt/idssys/nodemgmt +SCRIPT=${FOLDER}/nodemgmt-scripts.sh +TMPFOLDER=${FOLDER}/.tmp +LOGFOLDER=${FOLDER}/logs +LOGFILE=${LOGFOLDER}/logfile + +RENOTIFY=1800 + +[ ! -d ${TMPFOLDER} ] && mkdir ${TMPFOLDER} +[ ! -d ${LOGFOLDER} ] && mkdir ${LOGFOLDER} + + +NM_SRVCOPT=(start stop restart reload enable disable) + +#MYSQL_REPL_CHECK=( ) +#WEB_REPL_CHECK=(nginx www ssl lessh php pma pdnsa) +WEB_REPL_CHECK=( web-data ) +LB_REPL_CHECK=( lb-data ) +#NC_REPL_CHECK=(apache nextcloud-php nextcloud-data) +#NC_REPL_CHECK=( ) + +declare -A NGINX_PATHS +NGINX_PATHS['LB']='/opt/lb-data/nginx' +NGINX_PATHS['WEB']='/mnt/web-data/nginx' + +declare -A NM_HOSTNAMES +NM_HOSTNAMES['10.10.1.121']='Webserver-Node1' +NM_HOSTNAMES['10.10.1.122']='Webserver-Node2' +NM_HOSTNAMES['10.10.1.24']='Media-Manager' +NM_HOSTNAMES['10.10.1.51']='MySQL-Node1' +NM_HOSTNAMES['10.10.1.52']='MySQL-Node2' +NM_HOSTNAMES['10.10.10.81']='LoadBalancer-Node1' +NM_HOSTNAMES['10.10.10.82']='LoadBalancer-Node2' +NM_HOSTNAMES['10.10.10.10']='LAN-DNS-Node1' +NM_HOSTNAMES['10.10.10.11']='LAN-DNS-Node2' +NM_HOSTNAMES['10.2.1.2']='LAN-DNS-Offsite' +NM_HOSTNAMES['10.10.10.42']='Nextcloud-Server' +NM_HOSTNAMES['10.10.10.50']='MySQL-Manager' +NM_HOSTNAMES['10.10.0.240']='PowerWall-Server' +NM_HOSTNAMES['100.100.55.2']='WAN-DNS-Node1 (East)' +NM_HOSTNAMES['100.100.55.3']='WAN-DNS-Node2 (West)' +NM_HOSTNAMES['100.100.55.5']='WAN-Monitor-Node1' +NM_HOSTNAMES['3.214.174.170']='WAN-DNS-Node1 (East)' +NM_HOSTNAMES['54.213.11.227']='WAN-DNS-Node2 (West)' +NM_HOSTNAMES['10.10.1.28']='WAN-DNS-Node3 (Home)' +NM_HOSTNAMES['44.214.59.192']='WAN-Monitor-Node1' + +declare -i errtime + +IFS=, +NODE_TYPES=(${NODETYPES}) +MYSQL_HOSTS=(${MYSQLHOSTS}) +WEB_HOSTS=(${WEBHOSTS}) +LB_HOSTS=(${LBHOSTS}) +NC_HOSTS=(${NCHOSTS}) +WPD_HOSTS=(${WPDHOSTS}) +LPD_HOSTS=(${LPDHOSTS}) +MM_HOSTS=(${MMHOSTS}) +GL_HOSTS=(${GLHOSTS}) +PW_HOSTS=(${PWHOSTS}) +WM_HOSTS=(${WMHOSTS}) + +NODE_SERVICES=(${NODESERVICES}) +MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK}) +WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK}) +LB_SERVICES_CHECK=(${LBSERVICES_CHECK}) +NC_SERVICES_CHECK=(${NCSERVICES_CHECK}) +WPD_SERVICES_CHECK=(${WPDSERVICES_CHECK}) +LPD_SERVICES_CHECK=(${LPDSERVICES_CHECK}) +MM_SERVICES_CHECK=(${MMSERVICES_CHECK}) +PW_SERVICES_CHECK=(${PWSERVICES_CHECK}) +WM_SERVICES_CHECK=(${WMSERVICES_CHECK}) +LOCAL_SERVICES=(${LOCAL_SERVICES}) + +WEB_DOCKER=(${WEBDOCKER}) +LB_DOCKER=(${LBDOCKER}) +WM_DOCKER=(${WMDOCKER}) +PW_DOCKER=(${PWDOCKER}) + +WEB_SINGLESRVR_SERVICES=(${WEB_SINGLESRVRSERVICES}) +WEB_SINGLESRVR_DOCKERS=(${WEB_SINGLESRVRDOCKERS}) + +unset IFS + +declare -A NM_NODETYPES +NM_NODETYPES['MYSQL']='MySQL' +NM_NODETYPES['LB']='LoadBalance' +NM_NODETYPES['WEB']='Webserver' +NM_NODETYPES['NC']='Nextcloud' +NM_NODETYPES['WPD']='WAN PowerDNS' +NM_NODETYPES['LPD']='LAN PowerDNS' +NM_NODETYPES['MM']='Media Manager' +NM_NODETYPES['PW']='PowerWall' +NM_NODETYPES['WM']='WAN-Monitor' + +declare -A NM_SERVICES +NM_SERVICES['mysql']='MySQL' +NM_SERVICES['mariadb']='MariaDB (MySQL)' +NM_SERVICES['postgresql']='PostgreSQL' +NM_SERVICES['apache2']='Apache Web Server' +NM_SERVICES['nginx']='NGINX Web Server' +NM_SERVICES['gogs']='Gogs Git Repository' +NM_SERVICES['gitea']='Gitea Git Repository' +NM_SERVICES['haproxy']='HAProxy Load Balancer' +NM_SERVICES['keepalived']='Keepalived Load Balancer' +NM_SERVICES['maxscale']='MaxScale Load Balancer' +NM_SERVICES['cmon']='ClusterControl' +NM_SERVICES['cmon-events']='CC-Events Manager' +NM_SERVICES['cmon-cloud']='CC-Cloud Daemon' +NM_SERVICES['php5.6-fpm']='PHP 5.6 FPM' +NM_SERVICES['php7.2-fpm']='PHP 7.2 FPM' +NM_SERVICES['php7.3-fpm']='PHP 7.3 FPM' +NM_SERVICES['php7.4-fpm']='PHP 7.4 FPM' +NM_SERVICES['php8.0-fpm']='PHP 8.0 FPM' +NM_SERVICES['php8.1-fpm']='PHP 8.1 FPM' +NM_SERVICES['php8.2-fpm']='PHP 8.2 FPM' +NM_SERVICES['syncthing']='Syncthing' +NM_SERVICES['pdns']='PowerDNS Server' +NM_SERVICES['pdnsadmin']='PowerDNS-Admin' +NM_SERVICES['pdnsadmin.socket']='PowerDNS-Admin Socket' +NM_SERVICES['proxysql']='ProxySQL Load Balancer' +NM_SERVICES['radarr']='Radarr' +NM_SERVICES['sonarr']='Sonarr' +NM_SERVICES['jackett']='Jackett' +NM_SERVICES['prowlarr']='Prowlarr' +NM_SERVICES['qbittorrent']='qBittorrent' +NM_SERVICES['snap.tautulli.tautulli']='Tautulli' +NM_SERVICES['ombi']='Ombi' +NM_SERVICES['glusterd']='Gluster FileSystem' +NM_SERVICES['redis-server']='Redis Caching Server' +NM_SERVICES['mosquitto']='Mosquitto MQTT Broker' +NM_SERVICES['tailscaled']='Tailscale P2P VPN' +NM_SERVICES['monitor-power']='Server Power Monitor' +NM_SERVICES['monitor-temp']='Server Temp Monitor' + + +declare -A NM_DOCKERS +NM_DOCKERS['authelia']='Authelia' +NM_DOCKERS['phpmyadmin']='phpMyAdmin' +NM_DOCKERS['phppgadmin']='phpPgAdmin' +NM_DOCKERS['vaultwarden']='VaultWarden' +NM_DOCKERS['heimdall']='Heimdall' +NM_DOCKERS['uptime-kuma']='Uptime-Kuma' +NM_DOCKERS['nginx-proxy_manager']='Nginx Proxy Manager' + +declare -A NM_DOCKER_COMPOSE +NM_DOCKER_COMPOSE['authelia']='/mnt/web-data/authelia' +NM_DOCKER_COMPOSE['phpmyadmin']='/mnt/web-data/phpmyadmin' +NM_DOCKER_COMPOSE['phppgadmin']='/mnt/web-data/phppgadmin' +NM_DOCKER_COMPOSE['vaultwarden']='/mnt/web-data/vaultwarden' +NM_DOCKER_COMPOSE['heimdall']='/mnt/web-data/heimdall' +NM_DOCKER_COMPOSE['uptime-kuma']='/opt/uptime-kuma' +NM_DOCKER_COMPOSE['nginx-proxy_manager']='/opt/nginx-proxy' + +declare -A NM_SRVCOPTS +NM_SRVCOPTS['status']='Status' +NM_SRVCOPTS['start']='Start' +NM_SRVCOPTS['stop']='Stopp' +NM_SRVCOPTS['restart']='Restart' +NM_SRVCOPTS['reload']='Reload' +NM_SRVCOPTS['enable']='Enabl' +NM_SRVCOPTS['disable']='Disabl' +NM_SRVCOPTS['daemon-reload']='Daemon-Reload' + + +declare -A REPL_CHECKS +REPL_CHECKS['apache']='/etc/apache2' +REPL_CHECKS['nginx']='/mnt/web-data/nginx' +REPL_CHECKS['nginxproxy']='/opt/nginx-proxy' +REPL_CHECKS['lb-data']='/opt/lb-data' +REPL_CHECKS['ssl']='/mnt/web-data/letsencrypt' +REPL_CHECKS['lessh']='/home/le/.ssh' +REPL_CHECKS['php']='/mnt/web-data/php' +REPL_CHECKS['www']='/mnt/web-data/www' +REPL_CHECKS['web-data']='/mnt/web-data' +REPL_CHECKS['git']='/mnt/web-data/gitea' +REPL_CHECKS['gitssh']='/home/git' +REPL_CHECKS['pma']='/mnt/web-data/phpmyadmin' +REPL_CHECKS['pdnsa']='/mnt/web-data/powerdns-admin' +REPL_CHECKS['nextcloud-data']='/mnt/Nextcloud-Data' +REPL_CHECKS['nextcloud-php']='/etc/php' + +declare -A REPL_DESC +REPL_DESC['apache']='Apache Settings' +REPL_DESC['nginx']='NGINX Settings' +REPL_DESC['nginxproxy']='NGINX-Proxy Settings' +REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)' +REPL_DESC['ssl']='SSL Certificates' +REPL_DESC['lessh']='LetsEncrypt SSH Keys' +REPL_DESC['php']='PHP Settings' +REPL_DESC['www']='Webserver Files' +REPL_DESC['web-data']='Web Data (NGINX/PHP)' +REPL_DESC['git']='Gitea System' +REPL_DESC['gitssh']='Gitea SSH Keys' +REPL_DESC['pma']='phpMyAdmin Settings' +REPL_DESC['pdnsa']='PowerDNS Admin' +REPL_DESC['nextcloud-data']='Nextcloud Data' +REPL_DESC['nextcloud-php']='Nextcloud PHP Settings' + +declare -A BACKUP_ITEMS +BACKUP_ITEMS[nginx-settings]=/etc/nginx +BACKUP_ITEMS[nginx-logs]=/var/log/nginx +BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt +BACKUP_ITEMS[webserver-files]=/var/www +BACKUP_ITEMS[haproxy]=/etc/haproxy +BACKUP_ITEMS[keepalived]=/etc/keepalived +BACKUP_ITEMS[fail2ban]=/etc/fail2ban +# BACKUP_ITEMS[nginx]=/ + + + +GET-CHECKCERT-DOMAINS(){ + declare -A CHECKCERT_DOMAINS + IFS=$'\n' + for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do + HOST=${LINE%% *} + PORT=${LINE#* } + IFS=" " + CHECKCERT_DOMAINS[${HOST}]=${PORT} + done + unset IFS +} + +DISP_HEADER(){ + declare -i cw; declare -i spc1; declare -i c + if [ "$1" = true ]; then + clear + fi + echo + echo -e "${idsCL[LightGreen]} NodeMgmt - Node Monitoring & Management${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}" + DIVIDER . lightGreen + if [ "$2" = true ]; then + echo -e " Node hostname : ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME}${idsCL[Default]}${idsST[Reset]}" + if [ -z ${LOCAL_SERVICES+x} ]; then SERVICES=$( IFS=$','; echo "${NODE_SERVICES[*]}" ) + else SERVICES=$( IFS=$','; echo "${LOCAL_SERVICES[*]}" ) + fi + IFS=,; SERVICES2=(${SERVICES}); unset IFS + for srvc in "${SERVICES2[@]}"; do + c=0; cw=18; spc='' + spc1=${cw}-${#NM_SERVICES[${srvc}]} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_SERVICES[${srvc}]}$spc: " + if [ $(pgrep ${srvc} | wc -l) -gt "0" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" + fi + done + DIVIDER + if [ $(ls -1 ${FOLDER}/*.lastrun 2>/dev/null | wc -l) != 0 ];then + echo -e "${idsST[Bold]}Lastrun Items:${idsST[Reset]}" + for lastrun in ${FOLDER}/*.lastrun ; do + IFS='/'; lastrun_item=(${lastrun}); unset IFS + lastrun_item=$(echo ${lastrun_item[4]} | sed "s/.lastrun//g") + lastrun_item=$(echo ${lastrun_item} | sed "s/-/ /g") + lastrun_date=$(stat -c %y ${lastrun}) + IFS=' '; lastrun_date=(${lastrun_date}); unset IFS + IFS='.'; lastrun_time=(${lastrun_date[1]}); unset IFS + c=0; cw=18; spc='' + spc1=${cw}-${#lastrun_item} + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -e " ${lastrun_item~}${spc}: ${lastrun_date[0]} ${lastrun_time}" + done + + DIVIDER + fi + fi + echo +} + +CERT-CHECK(){ + TARGET="mysite.example.net"; + RECIPIENT="hostmaster@mysite.example.net"; + DAYS=7; + echo "checking if $TARGET expires in less than $DAYS days"; + expirationdate=$(date -d "$(: | openssl s_client -connect $TARGET:443 -servername $TARGET 2>/dev/null \ + | openssl x509 -text \ + | grep 'Not After' \ + |awk '{print $4,$5,$7}')" '+%s'); + in7days=$(($(date +%s) + (86400*$DAYS))); + if [ $in7days -gt $expirationdate ]; then + echo "KO - Certificate for $TARGET expires in less than $DAYS days, on $(date -d @$expirationdate '+%Y-%m-%d')" \ + | mail -s "Certificate expiration warning for $TARGET" $RECIPIENT ; + else + echo "OK - Certificate expires on $expirationdate"; + fi +} + +SENDNOTICE(){ + [ "${PUSHOVER_APP_TOKEN}" != "" ] && PUSH_TO_MOBILE "${2} + +$(date)" "${1}" ${3} & + + [ "${EMAIL_NOTICE}" != "" ] && echo -e "${1}\n\n$(date)" | mail -s "${2}" ${EMAIL_NOTICE} + +} + + + +