voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-03 23:05:23 -05:00
parent d02c6ccf36
commit c1fdda6e7d
4 changed files with 173 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
defaults.inc
View File

@@ -23,16 +23,6 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then
IFS=,
MYSQL_SERVICES_CHECK=(${MYSQLSERVICES_CHECK})
WEB_SERVICES_CHECK=(${WEBSERVICES_CHECK})
LB_SERVICES_CHECK=(${LBSERVICES_CHECK})
NC_SERVICES_CHECK=(${NCSERVICES_CHECK})
WPD_SERVICES_CHECK=(${WPDSERVICES_CHECK})
LPD_SERVICES_CHECK=(${LPDSERVICES_CHECK})
MM_SERVICES_CHECK=(${MMSERVICES_CHECK})
PW_SERVICES_CHECK=(${PWSERVICES_CHECK})
WM_SERVICES_CHECK=(${WMSERVICES_CHECK})
LOCAL_SERVICES=(${LOCAL_SERVICES})
WEB_DOCKER=(${WEBDOCKER})
@@ -53,22 +43,66 @@ if [ -f ${FOLDER}/defaults.local.inc ]; then
for ntype in "${NODE_TYPES[@]}"; do
arr ${ntype}_HOSTS
arr ${ntype}_SERVICES_CHECK
arr ${ntype}_DOCKERS_CHECK
arr ${ntype}_REPL_CHECK
IFS=,; var=(${NM_HOSTS[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_HOSTS $value
done
IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_SERVICES_CHECK $value
done
if [ "${NM_SERVICES_CHECK[${ntype}]}" != "" ]; then
IFS=,; var=(${NM_SERVICES_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_SERVICES_CHECK $value
done
else
declare "${ntype}_SERVICES_CHECK"=""
fi
if [ "${NM_DOCKERS_CHECK[${ntype}]}" != "" ]; then
IFS=,; var=(${NM_DOCKERS_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_DOCKERS_CHECK $value
done
else
declare "${ntype}_DOCKERS_CHECK"=""
fi
if [ "${NM_REPL_CHECK[${ntype}]}" != "" ]; then
IFS=,; var=(${NM_REPL_CHECK[${ntype}]}); unset IFS
for value in "${var[@]}"; do
arr_insert ${ntype}_REPL_CHECK $value
done
else
declare "${ntype}_DOCKERS_CHECK"=""
fi
done
fi
declare -A NM_SRVCOPTS
NM_SRVCOPTS['status']='Status'
NM_SRVCOPTS['start']='Start'
NM_SRVCOPTS['stop']='Stopp'
NM_SRVCOPTS['restart']='Restart'
NM_SRVCOPTS['reload']='Reload'
NM_SRVCOPTS['enable']='Enabl'
NM_SRVCOPTS['disable']='Disabl'
NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
declare -A NM_BACKUP_ITEMS
NM_BACKUP_ITEMS['nginx-settings']=/etc/nginx
NM_BACKUP_ITEMS['nginx-logs']=/var/log/nginx
NM_BACKUP_ITEMS['letsencrypt-certs']=/etc/letsencrypt
NM_BACKUP_ITEMS['webserver-files']=/var/www
NM_BACKUP_ITEMS['haproxy']=/etc/haproxy
NM_BACKUP_ITEMS['keepalived']=/etc/keepalived
NM_BACKUP_ITEMS['fail2ban']=/etc/fail2ban
# NM_BACKUP_ITEMS[nginx]=/
echo "HERE: "
# arr_get ${ntype}_HOSTS
for test in "${MYSQL_HOSTS[@]}"; do

166
defaults.local.inc
View File

@@ -37,22 +37,30 @@ NM_SERVICES_CHECK['WPD']=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,
NM_SERVICES_CHECK['WM']=nginx,tailscaled
NM_SERVICES_CHECK['WEB']=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
declare -A NM_DOCKERS_CHECK
NM_DOCKERS_CHECK['LB']=
NM_DOCKERS_CHECK['LPD']=
NM_DOCKERS_CHECK['MM']=
NM_DOCKERS_CHECK['MYSQL']=
NM_DOCKERS_CHECK['NC']=
NM_DOCKERS_CHECK['PW']=uptime-kuma
NM_DOCKERS_CHECK['WPD']=
NM_DOCKERS_CHECK['WM']=uptime-kuma
NM_DOCKERS_CHECK['WEB']=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia
declare -A NM_REPL_CHECK
NM_REPL_CHECK['LB']=lb-data
NM_REPL_CHECK['LPD']=
NM_REPL_CHECK['MM']=
NM_REPL_CHECK['MYSQL']=
NM_REPL_CHECK['NC']=
NM_REPL_CHECK['PW']=
NM_REPL_CHECK['WPD']=
NM_REPL_CHECK['WM']=
NM_REPL_CHECK['WEB']=web-data
MYSQLSERVICES_CHECK=mariadb,proxysql,keepalived
WEBSERVICES_CHECK=nginx,php5.6-fpm,php7.2-fpm,php7.3-fpm,php7.4-fpm,php8.2-fpm,gitea,keepalived
LBSERVICES_CHECK=nginx,haproxy,keepalived
NCSERVICES_CHECK=nginx,php8.1-fpm,redis-server
WPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,proxysql,tailscaled
LPDSERVICES_CHECK=pdns,nginx,mariadb,pdnsadmin,pdnsadmin.socket,keepalived
MMSERVICES_CHECK=bazarr,radarr,sonarr,prowlarr,qbittorrent,snap.tautulli.tautulli,ombi,nginx,php8.2-fpm
PWSERVICES_CHECK=mosquitto,monitor-power,monitor-temp
WMSERVICES_CHECK=nginx,tailscaled
WEBDOCKER=phpmyadmin,phppgadmin,vaultwarden,heimdall,authelia
WMDOCKER=uptime-kuma
PWDOCKER=uptime-kuma
#LBDOCKER=nginx-proxy_manager
WEB_SINGLESRVRSERVICES=gitea
WEB_SINGLESRVRDOCKERS=vaultwarden,heimdall,authelia
@@ -62,13 +70,6 @@ LB_NGINX_SITES='/opt/lb-data/nginx/sites-enabled'
WEB_NGINX_SITES='/mnt/web-data/nginx/sites-enabled'
#MYSQL_REPL_CHECK=( )
#WEB_REPL_CHECK=(nginx www ssl lessh php pma pdnsa)
WEB_REPL_CHECK=( web-data )
LB_REPL_CHECK=( lb-data )
#NC_REPL_CHECK=(apache nextcloud-php nextcloud-data)
#NC_REPL_CHECK=( )
declare -A NGINX_PATHS
NGINX_PATHS['LB']='/opt/lb-data/nginx'
NGINX_PATHS['WEB']='/mnt/web-data/nginx'
@@ -105,6 +106,7 @@ NM_NODETYPES['LPD']='LAN PowerDNS'
NM_NODETYPES['MM']='Media Manager'
NM_NODETYPES['PW']='PowerWall'
NM_NODETYPES['WM']='WAN-Monitor'
NM_NODETYPES['GL']='Gluster'
declare -A NM_SERVICE_DESC
NM_SERVICE_DESC['mysql']='MySQL'
@@ -128,9 +130,9 @@ NM_SERVICE_DESC['php8.0-fpm']='PHP 8.0 FPM'
NM_SERVICE_DESC['php8.1-fpm']='PHP 8.1 FPM'
NM_SERVICE_DESC['php8.2-fpm']='PHP 8.2 FPM'
NM_SERVICE_DESC['syncthing']='Syncthing'
NM_SERVICE_DESC['pdns']='PowerDNS Server'
NM_SERVICE_DESC['pdnsadmin']='PowerDNS-Admin'
NM_SERVICE_DESC['pdnsadmin.socket']='PowerDNS-Admin Socket'
NM_SERVICE_DESC['pdns']='(PowerDNS) DNS Serveice'
NM_SERVICE_DESC['pdnsadmin']='(PowerDNS) Admin Portal'
NM_SERVICE_DESC['pdnsadmin.socket']='(PowerDNS) Portal Socket'
NM_SERVICE_DESC['proxysql']='ProxySQL Load Balancer'
NM_SERVICE_DESC['radarr']='Radarr'
NM_SERVICE_DESC['sonarr']='Sonarr'
@@ -147,77 +149,59 @@ NM_SERVICE_DESC['monitor-power']='Server Power Monitor'
NM_SERVICE_DESC['monitor-temp']='Server Temp Monitor'
declare -A NM_DOCKERS
NM_DOCKERS['authelia']='Authelia'
NM_DOCKERS['phpmyadmin']='phpMyAdmin'
NM_DOCKERS['phppgadmin']='phpPgAdmin'
NM_DOCKERS['vaultwarden']='VaultWarden'
NM_DOCKERS['heimdall']='Heimdall'
NM_DOCKERS['uptime-kuma']='Uptime-Kuma'
NM_DOCKERS['nginx-proxy_manager']='Nginx Proxy Manager'
declare -A NM_DOCKER_DESC
NM_DOCKER_DESC['authelia']='Authelia'
NM_DOCKER_DESC['phpmyadmin']='phpMyAdmin'
NM_DOCKER_DESC['phppgadmin']='phpPgAdmin'
NM_DOCKER_DESC['vaultwarden']='VaultWarden'
NM_DOCKER_DESC['heimdall']='Heimdall'
NM_DOCKER_DESC['uptime-kuma']='Uptime-Kuma'
NM_DOCKER_DESC['nginx-proxy_manager']='Nginx Proxy Manager'
declare -A NM_DOCKER_COMPOSE
NM_DOCKER_COMPOSE['authelia']='/mnt/web-data/authelia'
NM_DOCKER_COMPOSE['phpmyadmin']='/mnt/web-data/phpmyadmin'
NM_DOCKER_COMPOSE['phppgadmin']='/mnt/web-data/phppgadmin'
NM_DOCKER_COMPOSE['vaultwarden']='/mnt/web-data/vaultwarden'
NM_DOCKER_COMPOSE['heimdall']='/mnt/web-data/heimdall'
NM_DOCKER_COMPOSE['uptime-kuma']='/opt/uptime-kuma'
NM_DOCKER_COMPOSE['nginx-proxy_manager']='/opt/nginx-proxy'
declare -A NM_DOCKER_COMPOSE_LOC
NM_DOCKER_COMPOSE_LOC['authelia']='/mnt/web-data/authelia'
NM_DOCKER_COMPOSE_LOC['phpmyadmin']='/mnt/web-data/phpmyadmin'
NM_DOCKER_COMPOSE_LOC['phppgadmin']='/mnt/web-data/phppgadmin'
NM_DOCKER_COMPOSE_LOC['vaultwarden']='/mnt/web-data/vaultwarden'
NM_DOCKER_COMPOSE_LOC['heimdall']='/mnt/web-data/heimdall'
NM_DOCKER_COMPOSE_LOC['uptime-kuma']='/opt/uptime-kuma'
NM_DOCKER_COMPOSE_LOC['nginx-proxy_manager']='/opt/nginx-proxy'
declare -A NM_REPL_CHECK_LOC
NM_REPL_CHECK_LOC['apache']='/etc/apache2'
NM_REPL_CHECK_LOC['nginx']='/mnt/web-data/nginx'
NM_REPL_CHECK_LOC['nginxproxy']='/opt/nginx-proxy'
NM_REPL_CHECK_LOC['lb-data']='/opt/lb-data'
NM_REPL_CHECK_LOC['ssl']='/mnt/web-data/letsencrypt'
NM_REPL_CHECK_LOC['lessh']='/home/le/.ssh'
NM_REPL_CHECK_LOC['php']='/mnt/web-data/php'
NM_REPL_CHECK_LOC['www']='/mnt/web-data/www'
NM_REPL_CHECK_LOC['web-data']='/mnt/web-data'
NM_REPL_CHECK_LOC['git']='/mnt/web-data/gitea'
NM_REPL_CHECK_LOC['gitssh']='/home/git'
NM_REPL_CHECK_LOC['pma']='/mnt/web-data/phpmyadmin'
NM_REPL_CHECK_LOC['pdnsa']='/mnt/web-data/powerdns-admin'
NM_REPL_CHECK_LOC['nextcloud-data']='/mnt/Nextcloud-Data'
NM_REPL_CHECK_LOC['nextcloud-php']='/etc/php'
declare -A NM_REPL_DESC
NM_REPL_DESC['apache']='Apache Settings'
NM_REPL_DESC['nginx']='NGINX Settings'
NM_REPL_DESC['nginxproxy']='NGINX-Proxy Settings'
NM_REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)'
NM_REPL_DESC['ssl']='SSL Certificates'
NM_REPL_DESC['lessh']='LetsEncrypt SSH Keys'
NM_REPL_DESC['php']='PHP Settings'
NM_REPL_DESC['www']='Webserver Files'
NM_REPL_DESC['web-data']='Web Data (NGINX/PHP)'
NM_REPL_DESC['git']='Gitea System'
NM_REPL_DESC['gitssh']='Gitea SSH Keys'
NM_REPL_DESC['pma']='phpMyAdmin Settings'
NM_REPL_DESC['pdnsa']='PowerDNS Admin'
NM_REPL_DESC['nextcloud-data']='Nextcloud Data'
NM_REPL_DESC['nextcloud-php']='Nextcloud PHP Settings'
declare -A NM_SRVCOPTS
NM_SRVCOPTS['status']='Status'
NM_SRVCOPTS['start']='Start'
NM_SRVCOPTS['stop']='Stopp'
NM_SRVCOPTS['restart']='Restart'
NM_SRVCOPTS['reload']='Reload'
NM_SRVCOPTS['enable']='Enabl'
NM_SRVCOPTS['disable']='Disabl'
NM_SRVCOPTS['daemon-reload']='Daemon-Reload'
declare -A REPL_CHECKS
REPL_CHECKS['apache']='/etc/apache2'
REPL_CHECKS['nginx']='/mnt/web-data/nginx'
REPL_CHECKS['nginxproxy']='/opt/nginx-proxy'
REPL_CHECKS['lb-data']='/opt/lb-data'
REPL_CHECKS['ssl']='/mnt/web-data/letsencrypt'
REPL_CHECKS['lessh']='/home/le/.ssh'
REPL_CHECKS['php']='/mnt/web-data/php'
REPL_CHECKS['www']='/mnt/web-data/www'
REPL_CHECKS['web-data']='/mnt/web-data'
REPL_CHECKS['git']='/mnt/web-data/gitea'
REPL_CHECKS['gitssh']='/home/git'
REPL_CHECKS['pma']='/mnt/web-data/phpmyadmin'
REPL_CHECKS['pdnsa']='/mnt/web-data/powerdns-admin'
REPL_CHECKS['nextcloud-data']='/mnt/Nextcloud-Data'
REPL_CHECKS['nextcloud-php']='/etc/php'
declare -A REPL_DESC
REPL_DESC['apache']='Apache Settings'
REPL_DESC['nginx']='NGINX Settings'
REPL_DESC['nginxproxy']='NGINX-Proxy Settings'
REPL_DESC['lb-data']='LB-Data (NGINX / SSL-Certs)'
REPL_DESC['ssl']='SSL Certificates'
REPL_DESC['lessh']='LetsEncrypt SSH Keys'
REPL_DESC['php']='PHP Settings'
REPL_DESC['www']='Webserver Files'
REPL_DESC['web-data']='Web Data (NGINX/PHP)'
REPL_DESC['git']='Gitea System'
REPL_DESC['gitssh']='Gitea SSH Keys'
REPL_DESC['pma']='phpMyAdmin Settings'
REPL_DESC['pdnsa']='PowerDNS Admin'
REPL_DESC['nextcloud-data']='Nextcloud Data'
REPL_DESC['nextcloud-php']='Nextcloud PHP Settings'
declare -A BACKUP_ITEMS
BACKUP_ITEMS[nginx-settings]=/etc/nginx
BACKUP_ITEMS[nginx-logs]=/var/log/nginx
BACKUP_ITEMS[letsencrypt-certs]=/etc/letsencrypt
BACKUP_ITEMS[webserver-files]=/var/www
BACKUP_ITEMS[haproxy]=/etc/haproxy
BACKUP_ITEMS[keepalived]=/etc/keepalived
BACKUP_ITEMS[fail2ban]=/etc/fail2ban
# BACKUP_ITEMS[nginx]=/

48
inc/status.inc
View File

@@ -24,10 +24,10 @@ STATUS(){
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
# echo "CHECK: ${NTYPE} - ${!PH} - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done
fi
fi
@@ -119,12 +119,12 @@ STATUS(){
for NTYPE in "${NODE_TYPES[@]}"; do
nid=1
dockers=${NTYPE}_DOCKER[@]
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
var=${NTYPE}_HOSTS[@]
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then
if [[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]; then
GOFORCHECK=false;
else
GOFORCHECK=true;
@@ -323,7 +323,7 @@ STATUS(){
########################
## DOCKER CHECK
########################
if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
@@ -336,39 +336,39 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is down, could not fix" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -482,11 +482,11 @@ STATUS(){
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
c=0; cw=30; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: "
fi
@@ -504,15 +504,15 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
@@ -548,10 +548,10 @@ STATUS(){
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl &
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else
${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi
done
fi
@@ -704,9 +704,9 @@ REPLCHECK(){
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl`
else
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\"`"
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done

46
nodemgmt-scripts.sh
View File

@@ -15,10 +15,10 @@ source /opt/idssys/nodemgmt/inc/sites.inc
CHECK_DOCKER_SERVICES(){
for NTYPE in "${NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKER[@]
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKER ]]; then
if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
@@ -54,15 +54,15 @@ CHECK_DOCKER_SERVICES(){
for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker}
fi
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
@@ -112,11 +112,11 @@ BACKUP(){
if [ ! -d ${BACKUP_FOLDER} ]; then
mkdir -p ${BACKUP_FOLDER}
fi
for item in "${!BACKUP_ITEMS[@]}"; do
for item in "${!NM_BACKUP_ITEMS[@]}"; do
c=0; spc=''; spc1=`expr 20 - ${#item}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "Backing up '${item}'${spc}"
if [ -d ${BACKUP_ITEMS[$item]} ]; then
tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1
if [ -d ${NM_BACKUP_ITEMS[$item]} ]; then
tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${NM_BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
@@ -502,8 +502,9 @@ SERVICE(){
fi
for NTYPE in "${NTS[@]}"; do
var1=${NTYPE}SERVICES_CHECK
if [[ "${!var1}" = *"${1}"* ]]; then
var1=${NTYPE}_SERVICES_CHECK
# if [[ "${!var1}" = *"${1}"* ]]; then
if [[ " ${!var1[*]} " =~ " ${1} " ]]; then
nid=1
if [ "${3}" != "" ]; then
var2=(${3})
@@ -587,7 +588,8 @@ SERVICE(){
HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
# if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -607,7 +609,8 @@ HAPROXY-CONFIG() {
done
echo
fi
if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
# if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -626,7 +629,8 @@ HAPROXY-CONFIG() {
nid=`expr $nid + 1`
done
fi
if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
# if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
if [[ " ${LB_SERVICES_CHECK[*]} " =~ " haproxy " ]]; then
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -664,7 +668,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
/bin/mount -a
sleep 5s
fi
if [ -f ${REPL_CHECKS["${1}-data"]}/mounted ]; then
if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
@@ -716,7 +720,8 @@ SINGLE_SERVER_SERVICE_CHECK(){
KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
# if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
if [[ " ${MYSQL_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
nid=1
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -736,7 +741,8 @@ KEEPALIVE-CONFIG() {
done
echo
fi
if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
# if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
if [[ " ${WEB_SERVICES_CHECK[*]} " =~ " keepalived " ]]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
@@ -1032,10 +1038,10 @@ DOWNLIST(){
if [[ $ditem == *"~"* ]] && [ "${NM_SERVICE_DESC[${item}]}" != "" ]; then
itemname=${NM_SERVICE_DESC[${item}]}
elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then
itemname="${NM_DOCKERS[${item}]} (docker)"
elif [[ $ditem == *"~"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then
itemname="${REPL_DESC[${item}]} (replication)"
elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKER_DESC[${item}]}" != "" ]; then
itemname="${NM_DOCKER_DESC[${item}]} (docker)"
elif [[ $ditem == *"~"* ]] && [ "${NM_REPL_DESC[${item}]}" != "" ]; then
itemname="${NM_REPL_DESC[${item}]} (replication)"
else
itemname='HOST DOWN';
[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"