voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-03 23:05:23 -05:00
parent d02c6ccf36
commit c1fdda6e7d
4 changed files with 173 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
defaults.inc
View File

@@ -23,16 +23,6 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then
IFS=, IFS=,
MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK})
WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK})
LB_SERVICES_CHECK=(${LBSERVICES_CHECK})
NC_SERVICES_CHECK=(${NCSERVICES_CHECK})
WPD_SERVICES_CHECK=(${WPDSERVICES_CHECK})
LPD_SERVICES_CHECK=(${LPDSERVICES_CHECK})
MM_SERVICES_CHECK=(${MMSERVICES_CHECK})
PW_SERVICES_CHECK=(${PWSERVICES_CHECK})
WM_SERVICES_CHECK=(${WMSERVICES_CHECK})
LOCAL_SERVICES=(${LOCAL_SERVICES}) LOCAL_SERVICES=(${LOCAL_SERVICES})
WEB_DOCKER=(${WEBDOCKER}) WEB_DOCKER=(${WEBDOCKER})
@@ -53,22 +43,66 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then
for ntype in "${NODE_TYPES[@]}"; do for ntype in "${NODE_TYPES[@]}"; do
arr ${ntype}_HOSTS arr ${ntype}_HOSTS
arr ${ntype}_SERVICES_CHECK arr ${ntype}_SERVICES_CHECK
arr ${ntype}_DOCKERS_CHECK
arr ${ntype}_REPL_CHECK
IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS
for value in "${var[@]}"; do for value in "${var[@]}"; do
arr_insert ${ntype}_HOSTS $value arr_insert ${ntype}_HOSTS $value
done done
IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS if [ "${NM_SERVICES_CHECK[${ntype}]}" != "" ]; then
for value in "${var[@]}"; do IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS
arr_insert ${ntype}_SERVICES_CHECK $value for value in "${var[@]}"; do
done arr_insert ${ntype}_SERVICES_CHECK $value
done
else
declare "${ntype}_SERVICES_CHECK"=""
fi
if [ "${NM_DOCKERS_CHECK[${ntype}]}" != "" ]; then
IFS=,; var=(${NM_DOCKERS_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_DOCKERS_CHECK $value
done
else
declare "${ntype}_DOCKERS_CHECK"=""
fi
if [ "${NM_REPL_CHECK[${ntype}]}" != "" ]; then
IFS=,; var=(${NM_REPL_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_REPL_CHECK $value
done
else
declare "${ntype}_DOCKERS_CHECK"=""
fi
done done
fi fi
declare -A NM_SRVCOPTS
NM_SRVCOPTS['status']='Status'
NM_SRVCOPTS['start']='Start'
NM_SRVCOPTS['stop']='Stopp'
NM_SRVCOPTS['restart']='Restart'
NM_SRVCOPTS['reload']='Reload'
NM_SRVCOPTS['enable']='Enabl'
NM_SRVCOPTS['disable']='Disabl'
NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
declare -A NM_BACKUP_ITEMS
NM_BACKUP_ITEMS['nginx-settings']=/etc/nginx
NM_BACKUP_ITEMS['nginx-logs']=/var/log/nginx
NM_BACKUP_ITEMS['letsencrypt-certs']=/etc/letsencrypt
NM_BACKUP_ITEMS['webserver-files']=/var/www
NM_BACKUP_ITEMS['haproxy']=/etc/haproxy
NM_BACKUP_ITEMS['keepalived']=/etc/keepalived
NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban
# NM_BACKUP_ITEMS[nginx]=/
echo "HERE: " echo "HERE: "
# arr_get ${ntype}_HOSTS # arr_get ${ntype}_HOSTS
for test in "${MYSQL_HOSTS[@]}"; do for test in "${MYSQL_HOSTS[@]}"; do

166
defaults.local.inc
View File

@@ -37,22 +37,30 @@ NM_SERVICES_CHECK['WPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,
NM_SERVICES_CHECK['WM']=nginx,tailscaled NM_SERVICES_CHECK['WM']=nginx,tailscaled
NM_SERVICES_CHECK['WEB']=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived NM_SERVICES_CHECK['WEB']=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
declare -A NM_DOCKERS_CHECK
NM_DOCKERS_CHECK['LB']=
NM_DOCKERS_CHECK['LPD']=
NM_DOCKERS_CHECK['MM']=
NM_DOCKERS_CHECK['MYSQL']=
NM_DOCKERS_CHECK['NC']=
NM_DOCKERS_CHECK['PW']=uptime-kuma
NM_DOCKERS_CHECK['WPD']=
NM_DOCKERS_CHECK['WM']=uptime-kuma
NM_DOCKERS_CHECK['WEB']=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia
declare -A NM_REPL_CHECK
NM_REPL_CHECK['LB']=lb-data
NM_REPL_CHECK['LPD']=
NM_REPL_CHECK['MM']=
NM_REPL_CHECK['MYSQL']=
NM_REPL_CHECK['NC']=
NM_REPL_CHECK['PW']=
NM_REPL_CHECK['WPD']=
NM_REPL_CHECK['WM']=
NM_REPL_CHECK['WEB']=web-data
MYSQLSERVICES_CHECK=mariadb,proxysql,keepalived
WEBSERVICES_CHECK=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
LBSERVICES_CHECK=nginx,haproxy,keepalived
NCSERVICES_CHECK=nginx,php8.1-fpm,redis-server
WPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,tailscaled
LPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,keepalived
MMSERVICES_CHECK=bazarr,radarr,sonarr,prowlarr,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm
PWSERVICES_CHECK=mosquitto,monitor-power,monitor-temp
WMSERVICES_CHECK=nginx,tailscaled
WEBDOCKER=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia
WMDOCKER=uptime-kuma
PWDOCKER=uptime-kuma
#LBDOCKER=nginx-proxy_manager
WEB_SINGLESRVRSERVICES=gitea WEB_SINGLESRVRSERVICES=gitea
WEB_SINGLESRVRDOCKERS=vaultwarden,heimdall,authelia WEB_SINGLESRVRDOCKERS=vaultwarden,heimdall,authelia
@@ -62,13 +70,6 @@ LB_NGINX_SITES='/opt/lb-data/nginx/sites-enabled'
WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled' WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled'
#MYSQL_REPL_CHECK=( )
#WEB_REPL_CHECK=(nginx www ssl lessh php pma pdnsa)
WEB_REPL_CHECK=( web-data )
LB_REPL_CHECK=( lb-data )
#NC_REPL_CHECK=(apache nextcloud-php nextcloud-data)
#NC_REPL_CHECK=( )
declare -A NGINX_PATHS declare -A NGINX_PATHS
NGINX_PATHS['LB']='/opt/lb-data/nginx' NGINX_PATHS['LB']='/opt/lb-data/nginx'
NGINX_PATHS['WEB']='/mnt/web-data/nginx' NGINX_PATHS['WEB']='/mnt/web-data/nginx'
@@ -105,6 +106,7 @@ NM_NODETYPES['LPD']='LAN PowerDNS'
NM_NODETYPES['MM']='Media Manager' NM_NODETYPES['MM']='Media Manager'
NM_NODETYPES['PW']='PowerWall' NM_NODETYPES['PW']='PowerWall'
NM_NODETYPES['WM']='WAN-Monitor' NM_NODETYPES['WM']='WAN-Monitor'
NM_NODETYPES['GL']='Gluster'
declare -A NM_SERVICE_DESC declare -A NM_SERVICE_DESC
NM_SERVICE_DESC['mysql']='MySQL' NM_SERVICE_DESC['mysql']='MySQL'
@@ -128,9 +130,9 @@ NM_SERVICE_DESC['php8.0-fpm']='PHP 8.0 FPM'
NM_SERVICE_DESC['php8.1-fpm']='PHP 8.1 FPM' NM_SERVICE_DESC['php8.1-fpm']='PHP 8.1 FPM'
NM_SERVICE_DESC['php8.2-fpm']='PHP 8.2 FPM' NM_SERVICE_DESC['php8.2-fpm']='PHP 8.2 FPM'
NM_SERVICE_DESC['syncthing']='Syncthing' NM_SERVICE_DESC['syncthing']='Syncthing'
NM_SERVICE_DESC['pdns']='PowerDNS Server' NM_SERVICE_DESC['pdns']='(PowerDNS) DNS Serveice'
NM_SERVICE_DESC['pdnsadmin']='PowerDNS-Admin' NM_SERVICE_DESC['pdnsadmin']='(PowerDNS) Admin Portal'
NM_SERVICE_DESC['pdnsadmin.socket']='PowerDNS-Admin Socket' NM_SERVICE_DESC['pdnsadmin.socket']='(PowerDNS) Portal Socket'
NM_SERVICE_DESC['proxysql']='ProxySQL Load Balancer' NM_SERVICE_DESC['proxysql']='ProxySQL Load Balancer'
NM_SERVICE_DESC['radarr']='Radarr' NM_SERVICE_DESC['radarr']='Radarr'
NM_SERVICE_DESC['sonarr']='Sonarr' NM_SERVICE_DESC['sonarr']='Sonarr'
@@ -147,77 +149,59 @@ NM_SERVICE_DESC['monitor-power']='Server Power Monitor'
NM_SERVICE_DESC['monitor-temp']='Server Temp Monitor' NM_SERVICE_DESC['monitor-temp']='Server Temp Monitor'
declare -A NM_DOCKERS declare -A NM_DOCKER_DESC
NM_DOCKERS['authelia']='Authelia' NM_DOCKER_DESC['authelia']='Authelia'
NM_DOCKERS['phpmyadmin']='phpMyAdmin' NM_DOCKER_DESC['phpmyadmin']='phpMyAdmin'
NM_DOCKERS['phppgadmin']='phpPgAdmin' NM_DOCKER_DESC['phppgadmin']='phpPgAdmin'
NM_DOCKERS['vaultwarden']='VaultWarden' NM_DOCKER_DESC['vaultwarden']='VaultWarden'
NM_DOCKERS['heimdall']='Heimdall' NM_DOCKER_DESC['heimdall']='Heimdall'
NM_DOCKERS['uptime-kuma']='Uptime-Kuma' NM_DOCKER_DESC['uptime-kuma']='Uptime-Kuma'
NM_DOCKERS['nginx-proxy_manager']='Nginx Proxy Manager' NM_DOCKER_DESC['nginx-proxy_manager']='Nginx Proxy Manager'
declare -A NM_DOCKER_COMPOSE declare -A NM_DOCKER_COMPOSE_LOC
NM_DOCKER_COMPOSE['authelia']='/mnt/web-data/authelia' NM_DOCKER_COMPOSE_LOC['authelia']='/mnt/web-data/authelia'
NM_DOCKER_COMPOSE['phpmyadmin']='/mnt/web-data/phpmyadmin' NM_DOCKER_COMPOSE_LOC['phpmyadmin']='/mnt/web-data/phpmyadmin'
NM_DOCKER_COMPOSE['phppgadmin']='/mnt/web-data/phppgadmin' NM_DOCKER_COMPOSE_LOC['phppgadmin']='/mnt/web-data/phppgadmin'
NM_DOCKER_COMPOSE['vaultwarden']='/mnt/web-data/vaultwarden' NM_DOCKER_COMPOSE_LOC['vaultwarden']='/mnt/web-data/vaultwarden'
NM_DOCKER_COMPOSE['heimdall']='/mnt/web-data/heimdall' NM_DOCKER_COMPOSE_LOC['heimdall']='/mnt/web-data/heimdall'
NM_DOCKER_COMPOSE['uptime-kuma']='/opt/uptime-kuma' NM_DOCKER_COMPOSE_LOC['uptime-kuma']='/opt/uptime-kuma'
NM_DOCKER_COMPOSE['nginx-proxy_manager']='/opt/nginx-proxy' NM_DOCKER_COMPOSE_LOC['nginx-proxy_manager']='/opt/nginx-proxy'
declare -A NM_REPL_CHECK_LOC
NM_REPL_CHECK_LOC['apache']='/etc/apache2'
NM_REPL_CHECK_LOC['nginx']='/mnt/web-data/nginx'
NM_REPL_CHECK_LOC['nginxproxy']='/opt/nginx-proxy'
NM_REPL_CHECK_LOC['lb-data']='/opt/lb-data'
NM_REPL_CHECK_LOC['ssl']='/mnt/web-data/letsencrypt'
NM_REPL_CHECK_LOC['lessh']='/home/le/.ssh'
NM_REPL_CHECK_LOC['php']='/mnt/web-data/php'
NM_REPL_CHECK_LOC['www']='/mnt/web-data/www'
NM_REPL_CHECK_LOC['web-data']='/mnt/web-data'
NM_REPL_CHECK_LOC['git']='/mnt/web-data/gitea'
NM_REPL_CHECK_LOC['gitssh']='/home/git'
NM_REPL_CHECK_LOC['pma']='/mnt/web-data/phpmyadmin'
NM_REPL_CHECK_LOC['pdnsa']='/mnt/web-data/powerdns-admin'
NM_REPL_CHECK_LOC['nextcloud-data']='/mnt/Nextcloud-Data'
NM_REPL_CHECK_LOC['nextcloud-php']='/etc/php'
declare -A NM_REPL_DESC
NM_REPL_DESC['apache']='Apache Settings'
NM_REPL_DESC['nginx']='NGINX Settings'
NM_REPL_DESC['nginxproxy']='NGINX-Proxy Settings'
NM_REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)'
NM_REPL_DESC['ssl']='SSL Certificates'
NM_REPL_DESC['lessh']='LetsEncrypt SSH Keys'
NM_REPL_DESC['php']='PHP Settings'
NM_REPL_DESC['www']='Webserver Files'
NM_REPL_DESC['web-data']='Web Data (NGINX/PHP)'
NM_REPL_DESC['git']='Gitea System'
NM_REPL_DESC['gitssh']='Gitea SSH Keys'
NM_REPL_DESC['pma']='phpMyAdmin Settings'
NM_REPL_DESC['pdnsa']='PowerDNS Admin'
NM_REPL_DESC['nextcloud-data']='Nextcloud Data'
NM_REPL_DESC['nextcloud-php']='Nextcloud PHP Settings'
declare -A NM_SRVCOPTS
NM_SRVCOPTS['status']='Status'
NM_SRVCOPTS['start']='Start'
NM_SRVCOPTS['stop']='Stopp'
NM_SRVCOPTS['restart']='Restart'
NM_SRVCOPTS['reload']='Reload'
NM_SRVCOPTS['enable']='Enabl'
NM_SRVCOPTS['disable']='Disabl'
NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
declare -A REPL_CHECKS
REPL_CHECKS['apache']='/etc/apache2'
REPL_CHECKS['nginx']='/mnt/web-data/nginx'
REPL_CHECKS['nginxproxy']='/opt/nginx-proxy'
REPL_CHECKS['lb-data']='/opt/lb-data'
REPL_CHECKS['ssl']='/mnt/web-data/letsencrypt'
REPL_CHECKS['lessh']='/home/le/.ssh'
REPL_CHECKS['php']='/mnt/web-data/php'
REPL_CHECKS['www']='/mnt/web-data/www'
REPL_CHECKS['web-data']='/mnt/web-data'
REPL_CHECKS['git']='/mnt/web-data/gitea'
REPL_CHECKS['gitssh']='/home/git'
REPL_CHECKS['pma']='/mnt/web-data/phpmyadmin'
REPL_CHECKS['pdnsa']='/mnt/web-data/powerdns-admin'
REPL_CHECKS['nextcloud-data']='/mnt/Nextcloud-Data'
REPL_CHECKS['nextcloud-php']='/etc/php'
declare -A REPL_DESC
REPL_DESC['apache']='Apache Settings'
REPL_DESC['nginx']='NGINX Settings'
REPL_DESC['nginxproxy']='NGINX-Proxy Settings'
REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)'
REPL_DESC['ssl']='SSL Certificates'
REPL_DESC['lessh']='LetsEncrypt SSH Keys'
REPL_DESC['php']='PHP Settings'
REPL_DESC['www']='Webserver Files'
REPL_DESC['web-data']='Web Data (NGINX/PHP)'
REPL_DESC['git']='Gitea System'
REPL_DESC['gitssh']='Gitea SSH Keys'
REPL_DESC['pma']='phpMyAdmin Settings'
REPL_DESC['pdnsa']='PowerDNS Admin'
REPL_DESC['nextcloud-data']='Nextcloud Data'
REPL_DESC['nextcloud-php']='Nextcloud PHP Settings'
declare -A BACKUP_ITEMS
BACKUP_ITEMS[nginx-settings]=/etc/nginx
BACKUP_ITEMS[nginx-logs]=/var/log/nginx
BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt
BACKUP_ITEMS[webserver-files]=/var/www
BACKUP_ITEMS[haproxy]=/etc/haproxy
BACKUP_ITEMS[keepalived]=/etc/keepalived
BACKUP_ITEMS[fail2ban]=/etc/fail2ban
# BACKUP_ITEMS[nginx]=/

48
inc/status.inc
View File

@@ -24,10 +24,10 @@ STATUS(){
var=${NTYPE}_REPL_CHECK[@] var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}" # echo "CHECK: ${NTYPE} - ${!PH} - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl ${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S` daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" & ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done done
fi fi
fi fi
@@ -119,12 +119,12 @@ STATUS(){
for NTYPE in "${NODE_TYPES[@]}"; do for NTYPE in "${NODE_TYPES[@]}"; do
nid=1 nid=1
dockers=${NTYPE}_DOCKER[@] dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@] hosts=${NTYPE}_HOSTS[@]
var=${NTYPE}_HOSTS[@] var=${NTYPE}_HOSTS[@]
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then if [[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]; then
GOFORCHECK=false; GOFORCHECK=false;
else else
GOFORCHECK=true; GOFORCHECK=true;
@@ -323,7 +323,7 @@ STATUS(){
######################## ########################
## DOCKER CHECK ## DOCKER CHECK
######################## ########################
if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo echo
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}" echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
@@ -336,39 +336,39 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc='' c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else else
spc=' ' spc=' '
fi fi
echo -en " ${NM_DOCKERS[${docker}]}$spc: " echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1 ${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not running, starting now" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
else else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1 ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
fi fi
sleep 10s sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is fixed" SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.* rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
else else
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is down, could not fix" 1 SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is down, could not fix" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
else else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down) errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then if [ $errtime -gt ${RENOTIFY} ]; then
@@ -482,11 +482,11 @@ STATUS(){
for rcheck in "${!var2}"; do for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done c=0; cw=30; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else else
spc=' ' spc=' '
fi fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: " echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: "
fi fi
@@ -504,15 +504,15 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down touch ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1 SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi fi
else else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}" [ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE} echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi fi
fi fi
done done
@@ -548,10 +548,10 @@ STATUS(){
daterun=`date +%Y-%m-%d-%H-%M-%S` daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl & rm -f ${FOLDER}/test.repl &
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else else
${PH_CMD} rm -f ${FOLDER}/test.repl & ${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl & ${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi fi
done done
fi fi
@@ -704,9 +704,9 @@ REPLCHECK(){
checked=false checked=false
until [ "${checked}" == "" ]; do until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then if [ "${PH_CMD}" == "" ]; then
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl`
else else
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\"`"
fi fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break [ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done done

46
nodemgmt-scripts.sh
View File

@@ -15,10 +15,10 @@ source /opt/idssys/nodemgmt/inc/sites.inc
CHECK_DOCKER_SERVICES(){ CHECK_DOCKER_SERVICES(){
for NTYPE in "${NODE_TYPES[@]}"; do for NTYPE in "${NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKER[@] dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@] hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKER ]]; then if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}" echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
@@ -54,15 +54,15 @@ CHECK_DOCKER_SERVICES(){
for docker in "${!dockers}"; do for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc='' c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}` spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKERS[${docker}]}$spc: " echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker} ${NCMD} docker rm ${docker}
fi fi
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
sleep 10s sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then if [ "${ST_ACTION}" != "check" ]; then
@@ -112,11 +112,11 @@ BACKUP(){
if [ ! -d ${BACKUP_FOLDER} ]; then if [ ! -d ${BACKUP_FOLDER} ]; then
mkdir -p ${BACKUP_FOLDER} mkdir -p ${BACKUP_FOLDER}
fi fi
for item in "${!BACKUP_ITEMS[@]}"; do for item in "${!NM_BACKUP_ITEMS[@]}"; do
c=0; spc=''; spc1=`expr 20 - ${#item}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done c=0; spc=''; spc1=`expr 20 - ${#item}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "Backing up '${item}'${spc}" echo -en "Backing up '${item}'${spc}"
if [ -d ${BACKUP_ITEMS[$item]} ]; then if [ -d ${NM_BACKUP_ITEMS[$item]} ]; then
tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1 tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${NM_BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}" echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else else
echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}" echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
@@ -502,8 +502,9 @@ SERVICE(){
fi fi
for NTYPE in "${NTS[@]}"; do for NTYPE in "${NTS[@]}"; do
var1=${NTYPE}SERVICES_CHECK var1=${NTYPE}_SERVICES_CHECK
if [[ "${!var1}" = *"${1}"* ]]; then # if [[ "${!var1}" = *"${1}"* ]]; then
if [[ " ${!var1[*]} " =~ " ${1} " ]]; then
nid=1 nid=1
if [ "${3}" != "" ]; then if [ "${3}" != "" ]; then
var2=(${3}) var2=(${3})
@@ -587,7 +588,8 @@ SERVICE(){
HAPROXY-CONFIG() { HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}" echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then # if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1 nid=1
for nip in "${MYSQL_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -607,7 +609,8 @@ HAPROXY-CONFIG() {
done done
echo echo
fi fi
if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then # if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1 nid=1
for nip in "${WEB_HOSTS[@]}"; do for nip in "${WEB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -626,7 +629,8 @@ HAPROXY-CONFIG() {
nid=`expr $nid + 1` nid=`expr $nid + 1`
done done
fi fi
if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then # if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${LB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1 nid=1
for nip in "${LB_HOSTS[@]}"; do for nip in "${LB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -664,7 +668,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
/bin/mount -a /bin/mount -a
sleep 5s sleep 5s
fi fi
if [ -f ${REPL_CHECKS["${1}-data"]}/mounted ]; then if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then
for stc in "${!servicestocheck}"; do for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
@@ -716,7 +720,8 @@ SINGLE_SERVER_SERVICE_CHECK(){
KEEPALIVE-CONFIG() { KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}" echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
DIVIDER false yellow DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then # if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
nid=1 nid=1
for nip in "${MYSQL_HOSTS[@]}"; do for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -736,7 +741,8 @@ KEEPALIVE-CONFIG() {
done done
echo echo
fi fi
if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then # if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
nid=1 nid=1
for nip in "${WEB_HOSTS[@]}"; do for nip in "${WEB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -1032,10 +1038,10 @@ DOWNLIST(){
if [[ $ditem == *"~"* ]] && [ "${NM_SERVICE_DESC[${item}]}" != "" ]; then if [[ $ditem == *"~"* ]] && [ "${NM_SERVICE_DESC[${item}]}" != "" ]; then
itemname=${NM_SERVICE_DESC[${item}]} itemname=${NM_SERVICE_DESC[${item}]}
elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKER_DESC[${item}]}" != "" ]; then
itemname="${NM_DOCKERS[${item}]} (docker)" itemname="${NM_DOCKER_DESC[${item}]} (docker)"
elif [[ $ditem == *"~"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then elif [[ $ditem == *"~"* ]] && [ "${NM_REPL_DESC[${item}]}" != "" ]; then
itemname="${REPL_DESC[${item}]} (replication)" itemname="${NM_REPL_DESC[${item}]} (replication)"
else else
itemname='HOST DOWN'; itemname='HOST DOWN';
[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}" [ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"