voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-06 22:12:38 -05:00
parent 86027b7bce
commit 0b586f400e
3 changed files with 432 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
defaults.inc
View File

@@ -1,5 +1,5 @@
VERS='4.8.16-07042023'
VERS='4.9.1-07062023'
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns '
CERT_DAEMON='/snap/bin/certbot'

430
inc/services.inc Executable file
View File

@@ -0,0 +1,430 @@
#!/usr/bin/env bash
SINGLE_SERVER_SERVICE_CHECK(){
servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
hosts=${1^^}_HOSTS[@]
if [ "${!servicestocheck}" != "" ] || [ "${!dockerstocheck}" != "" ]; then
local_ips=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1);
local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1)
if [[ ${local_ips} = *10.10.1.120* ]]; then
if [ ! -f /mnt/web-data/mounted ]; then
/bin/mount -a
sleep 5s
fi
if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
for nip in "${!hosts}"; do
if [[ "${local_ip}" != *"${nip}"* ]]; then
ssh root@${nip} /bin/systemctl stop ${stc}
fi
done
sleep 3s
/bin/systemctl start ${stc} &
fi
done
for docker in "${!dockerstocheck}"; do
for nip in "${!hosts}"; do
if [[ "${local_ip}" = *"${nip}"* ]]; then
[ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1
[ "${docker}" == "authelia" ] && [ ! "$(/usr/bin/docker ps -q -f name=${docker}_redis)" ] && /usr/bin/docker start ${docker}_redis & >/dev/null 2>&1
# [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "starting local" || echo "already started local"
else
[ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && ssh root@${nip} /usr/bin/docker stop ${docker} & >/dev/null 2>&1
[ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker}_redis)" ] && ssh root@${nip} /usr/bin/docker stop ${docker}_redis & >/dev/null 2>&1
# [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping on $nip" || echo "already stopped on $nip"
fi
done
done
fi
else
for stc in "${!servicestocheck}"; do
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
/bin/systemctl stop ${stc} &
done
for docker in "${!dockerstocheck}"; do
[ "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker stop ${docker} & >/dev/null 2>&1
# [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping" || echo "already stopped"
done
fi
if [ "$(systemctl is-active nginx)" != "active" ]; then
${NM_SCRIPT} service web restart ${local_ip}
fi
fi
}
SERVICE(){
#var=NC_HOSTS[@]
if [ "$1" = "glusterd" ]; then
for nip in "${GL_HOSTS[@]}"; do
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD systemctl $2 $1 > /dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then
echo
sleep 5s
for nip in "${GL_HOSTS[@]}"; do
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD mount -a > /dev/null 2>&1
if [ "${3}" != "q" ]; then
# if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
if [[ " ${NC_HOSTS[*]} " =~ " ${nip} " ]]; then
mntpath="/mnt/Nextcloud-Data"
# elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
elif [[ " ${WEB_HOSTS[*]} " =~ " ${nip} " ]]; then
mntpath="/mnt/web-data"
fi
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
if [ "$nip" = "10.10.1.43" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
mntpath="/mnt/Nextcloud-Data"
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
fi
else
if [ "${4}" != "all" ]; then
if [ "${RUN_NODE_TYPE}" != "" ]; then
NT_HOSTS=${RUN_NODE_TYPE}_HOSTS[@]
NT_HOST=${RUN_NODE_TYPE}_HOSTS[0]
NTS=("${RUN_NODE_TYPE}");
else
NTS=(${NM_NODE_TYPES[*]})
fi
else
NTS=(${NM_NODE_TYPES[*]})
fi
# if [ "${RUN_NODE_TYPE}" == "" ]; then
#
# fi
if [ "${3}" != "q" ]; then
if [ "${NM_SERVICE_DESC[${1}]}" = "" ]; then
echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
exit 1
fi
if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then
echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}"
exit 1
fi
if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then
TACT='restart'
else
TACT="${2}"
fi
echo
echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICE_DESC[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
fi
if [ "${1}" = "nginx" ] && ([ "${3}" == "" ] || [ "${3}" == "all" ]); then
[ "${RUN_NODE_TYPE}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] &&
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}"
fi
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD=""
else
PH_CMD="ssh root@${!NT_HOST}"
fi
${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl
# ${PH_CMD} "echo -e \"Service ${1} ${2}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl"
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "true" ]; do
checked="`${PH_CMD} ssh root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl)) -le 90 ] && echo true`"
done
fi
done
${PH_CMD} rm -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl
fi
fi
elif [ "${1}" = "apache2-x" ]; then
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}"
fi
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD="ssh root@localhost"
else
PH_CMD="ssh root@${!NT_HOST}"
fi
echo "HERE: '$PH_CMD'"
${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl"
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "" ]; do
checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`"
done
fi
fi
done
${PH_CMD} rm -f /etc/apache2/test.repl
fi
fi
fi
if [ "${3}" != "q" ]; then
echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
echo
fi
for NTYPE in "${NTS[@]}"; do
var1=${NTYPE}_SERVICES_CHECK[@]
# if [[ "${!var1}" = *"${1}"* ]]; then
if [[ " ${!var1} " =~ " ${1} " ]]; then
nid=1
if [ "${3}" != "" ]; then
var2=(${3})
sethost=true;
else
var2=${NTYPE}_HOSTS[@]
IFS=' '
var2=(${!var2})
unset IFS
sethost=false
echo $var2
fi
for nip in "${var2[@]}"; do
# if [[ "${RUN_NODE_IP}" == *"${3}"* ]]; then GO=true;
# elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true;
# else GO=false;
# fi
# if [ "${GO}" == "true" ]; then
if [ $sethost == true ]; then
nodename=${NM_HOSTNAMES[${3}]}
else
nodename="${NM_HOSTNAMES[${nip}]}[${nip}]"
fi
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
NOGOCHK=true;
# if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then
if [ "${1}" == "gitea" ]; then
if [[ $($NCMD ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) != *10.10.1.120* ]]; then
NOGOCHK=false;
fi
fi
if [ "${NOGOCHK}" == "true" ]; then
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
[ "${1}" = "pdnsadmin" ] && srva="pdnsadmin.socket pdnsadmin.service" || srva=${1}
$NCMD systemctl ${2} ${srva} >/dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
fi
# fi
nid=`expr $nid + 1`
# if [ "${1}" = "gitea" ]; then
# break
# fi
done
fi
done
fi
if [ "${3}" != "q" ]; then
echo
echo -e "${idsCL[Green]}${NM_SERVICE_DESC[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
echo
fi
}
CHECK_DOCKER_SERVICES(){
for NTYPE in "${NM_NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${!hosts}"; do
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
uptime=`${NCMD} uptime -p`
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker}
fi
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
fi
done
else
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
fi
done
}

424
nodemgmt-scripts.sh
View File

@@ -11,95 +11,10 @@ source /opt/idssys/nodemgmt/defaults.inc
source /opt/idssys/nodemgmt/inc/status.inc
source /opt/idssys/nodemgmt/inc/certs.inc
source /opt/idssys/nodemgmt/inc/sites.inc
source /opt/idssys/nodemgmt/inc/services.inc
CHECK_DOCKER_SERVICES(){
for NTYPE in "${NM_NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${!hosts}"; do
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
uptime=`${NCMD} uptime -p`
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker}
fi
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
fi
done
else
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
fi
done
}
BACKUP(){
BACKUP_FOLDER=/opt/idssys/backups/node-backup/$(date +%m-%d-%y)/$(date +%H-%M-%S)
@@ -282,282 +197,6 @@ SET-PERMISSIONS(){
fi
}
SERVICE(){
#var=NC_HOSTS[@]
if [ "$1" = "glusterd" ]; then
for nip in "${GL_HOSTS[@]}"; do
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD systemctl $2 $1 > /dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then
echo
sleep 5s
for nip in "${GL_HOSTS[@]}"; do
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD mount -a > /dev/null 2>&1
if [ "${3}" != "q" ]; then
# if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
if [[ " ${NC_HOSTS[*]} " =~ " ${nip} " ]]; then
mntpath="/mnt/Nextcloud-Data"
# elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
elif [[ " ${WEB_HOSTS[*]} " =~ " ${nip} " ]]; then
mntpath="/mnt/web-data"
fi
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
if [ "$nip" = "10.10.1.43" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
mntpath="/mnt/Nextcloud-Data"
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
fi
else
if [ "${4}" != "all" ]; then
if [ "${RUN_NODE_TYPE}" != "" ]; then
NT_HOSTS=${RUN_NODE_TYPE}_HOSTS[@]
NT_HOST=${RUN_NODE_TYPE}_HOSTS[0]
NTS=("${RUN_NODE_TYPE}");
else
NTS=(${NM_NODE_TYPES[*]})
fi
else
NTS=(${NM_NODE_TYPES[*]})
fi
# if [ "${RUN_NODE_TYPE}" == "" ]; then
#
# fi
if [ "${3}" != "q" ]; then
if [ "${NM_SERVICE_DESC[${1}]}" = "" ]; then
echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
exit 1
fi
if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then
echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}"
exit 1
fi
if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then
TACT='restart'
else
TACT="${2}"
fi
echo
echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICE_DESC[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
fi
if [ "${1}" = "nginx" ] && ([ "${3}" == "" ] || [ "${3}" == "all" ]); then
[ "${RUN_NODE_TYPE}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] &&
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}"
fi
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD=""
else
PH_CMD="ssh root@${!NT_HOST}"
fi
${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl
# ${PH_CMD} "echo -e \"Service ${1} ${2}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl"
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "true" ]; do
checked="`${PH_CMD} ssh root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl)) -le 90 ] && echo true`"
done
fi
done
${PH_CMD} rm -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl
fi
fi
elif [ "${1}" = "apache2-x" ]; then
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}"
fi
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD="ssh root@localhost"
else
PH_CMD="ssh root@${!NT_HOST}"
fi
echo "HERE: '$PH_CMD'"
${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl"
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "" ]; do
checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`"
done
fi
fi
done
${PH_CMD} rm -f /etc/apache2/test.repl
fi
fi
fi
if [ "${3}" != "q" ]; then
echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
echo
fi
for NTYPE in "${NTS[@]}"; do
var1=${NTYPE}_SERVICES_CHECK[@]
# if [[ "${!var1}" = *"${1}"* ]]; then
if [[ " ${!var1} " =~ " ${1} " ]]; then
nid=1
if [ "${3}" != "" ]; then
var2=(${3})
sethost=true;
else
var2=${NTYPE}_HOSTS[@]
IFS=' '
var2=(${!var2})
unset IFS
sethost=false
echo $var2
fi
for nip in "${var2[@]}"; do
# if [[ "${RUN_NODE_IP}" == *"${3}"* ]]; then GO=true;
# elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true;
# else GO=false;
# fi
# if [ "${GO}" == "true" ]; then
if [ $sethost == true ]; then
nodename=${NM_HOSTNAMES[${3}]}
else
nodename="${NM_HOSTNAMES[${nip}]}[${nip}]"
fi
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
NOGOCHK=true;
# if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then
if [ "${1}" == "gitea" ]; then
if [[ $($NCMD ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) != *10.10.1.120* ]]; then
NOGOCHK=false;
fi
fi
if [ "${NOGOCHK}" == "true" ]; then
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
[ "${1}" = "pdnsadmin" ] && srva="pdnsadmin.socket pdnsadmin.service" || srva=${1}
$NCMD systemctl ${2} ${srva} >/dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
fi
# fi
nid=`expr $nid + 1`
# if [ "${1}" = "gitea" ]; then
# break
# fi
done
fi
done
fi
if [ "${3}" != "q" ]; then
echo
echo -e "${idsCL[Green]}${NM_SERVICE_DESC[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
echo
fi
}
HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow
@@ -628,67 +267,6 @@ HAPROXY-CONFIG() {
echo
}
SINGLE_SERVER_SERVICE_CHECK(){
servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
hosts=${1^^}_HOSTS[@]
if [ "${!servicestocheck}" != "" ] || [ "${!dockerstocheck}" != "" ]; then
local_ips=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1);
local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1)
if [[ ${local_ips} = *10.10.1.120* ]]; then
if [ ! -f /mnt/web-data/mounted ]; then
/bin/mount -a
sleep 5s
fi
if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
for nip in "${!hosts}"; do
if [[ "${local_ip}" != *"${nip}"* ]]; then
ssh root@${nip} /bin/systemctl stop ${stc}
fi
done
sleep 3s
/bin/systemctl start ${stc} &
fi
done
for docker in "${!dockerstocheck}"; do
for nip in "${!hosts}"; do
if [[ "${local_ip}" = *"${nip}"* ]]; then
[ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1
[ "${docker}" == "authelia" ] && [ ! "$(/usr/bin/docker ps -q -f name=${docker}_redis)" ] && /usr/bin/docker start ${docker}_redis & >/dev/null 2>&1
# [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "starting local" || echo "already started local"
else
[ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && ssh root@${nip} /usr/bin/docker stop ${docker} & >/dev/null 2>&1
[ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker}_redis)" ] && ssh root@${nip} /usr/bin/docker stop ${docker}_redis & >/dev/null 2>&1
# [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping on $nip" || echo "already stopped on $nip"
fi
done
done
fi
else
for stc in "${!servicestocheck}"; do
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
/bin/systemctl stop ${stc} &
done
for docker in "${!dockerstocheck}"; do
[ "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker stop ${docker} & >/dev/null 2>&1
# [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping" || echo "already stopped"
done
fi
if [ "$(systemctl is-active nginx)" != "active" ]; then
${NM_SCRIPT} service web restart ${local_ip}
fi
fi
}
KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"