diff --git a/defaults.inc b/defaults.inc index 82d37706..0920c344 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,5 +1,5 @@ -VERS='4.8.16-07042023' +VERS='4.9.1-07062023' noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns ' CERT_DAEMON='/snap/bin/certbot' diff --git a/inc/services.inc b/inc/services.inc new file mode 100755 index 00000000..59c754a1 --- /dev/null +++ b/inc/services.inc @@ -0,0 +1,430 @@ +#!/usr/bin/env bash +SINGLE_SERVER_SERVICE_CHECK(){ + servicestocheck=${1^^}_SINGLESRVR_SERVICES[@] + dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@] + hosts=${1^^}_HOSTS[@] + + if [ "${!servicestocheck}" != "" ] || [ "${!dockerstocheck}" != "" ]; then + local_ips=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1); + local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1) + if [[ ${local_ips} = *10.10.1.120* ]]; then + if [ ! -f /mnt/web-data/mounted ]; then + /bin/mount -a + sleep 5s + fi + if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then + for stc in "${!servicestocheck}"; do + if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then + SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" + echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE} + + [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" + for nip in "${!hosts}"; do + if [[ "${local_ip}" != *"${nip}"* ]]; then + ssh root@${nip} /bin/systemctl stop ${stc} + fi + done + sleep 3s + /bin/systemctl start ${stc} & + fi + done + for docker in "${!dockerstocheck}"; do + for nip in "${!hosts}"; do + if [[ "${local_ip}" = *"${nip}"* ]]; then + [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1 + [ "${docker}" == "authelia" ] && [ ! "$(/usr/bin/docker ps -q -f name=${docker}_redis)" ] && /usr/bin/docker start ${docker}_redis & >/dev/null 2>&1 + # [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "starting local" || echo "already started local" + else + [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && ssh root@${nip} /usr/bin/docker stop ${docker} & >/dev/null 2>&1 + [ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker}_redis)" ] && ssh root@${nip} /usr/bin/docker stop ${docker}_redis & >/dev/null 2>&1 + # [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping on $nip" || echo "already stopped on $nip" + fi + done + done + fi + + + else + for stc in "${!servicestocheck}"; do + [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" + /bin/systemctl stop ${stc} & + done + for docker in "${!dockerstocheck}"; do + [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker stop ${docker} & >/dev/null 2>&1 + # [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping" || echo "already stopped" + done + fi + + if [ "$(systemctl is-active nginx)" != "active" ]; then + ${NM_SCRIPT} service web restart ${local_ip} + fi + fi +} + +SERVICE(){ + #var=NC_HOSTS[@] + + if [ "$1" = "glusterd" ]; then + + for nip in "${GL_HOSTS[@]}"; do + if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then + nip='localhost ' + NCMD='' + else + NCMD="ssh root@${nip}" + fi + + if [ "${3}" != "q" ]; then + echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}" + fi + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + #$NCMD systemctl $2 $1 > /dev/null 2>&1 + if [ "${3}" != "q" ]; then + if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + elif [ "$2" = "stop" ]; then + echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}" + else + echo -e "${idsCL[Red]}ERROR${idsCL[Default]}" + fi + fi + elif [ "${3}" != "q" ]; then + echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" + fi + + done + + if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then + + echo + sleep 5s + + for nip in "${GL_HOSTS[@]}"; do + if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then + nip='localhost ' + NCMD='' + else + NCMD="ssh root@${nip}" + fi + + if [ "${3}" != "q" ]; then + echo -en "Mounting drive on '$nip'... ${idsCL[Default]}" + fi + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + #$NCMD mount -a > /dev/null 2>&1 + if [ "${3}" != "q" ]; then + # if [[ "${NCHOSTS}" == *"${nip}"* ]]; then + if [[ " ${NC_HOSTS[*]} " =~ " ${nip} " ]]; then + mntpath="/mnt/Nextcloud-Data" + # elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then + elif [[ " ${WEB_HOSTS[*]} " =~ " ${nip} " ]]; then + mntpath="/mnt/web-data" + fi + ${NCMD} "test -e ${mntpath}/mounted" + if [ $? -eq 0 ]; then + echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}" + else + echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}" + fi + fi + + if [ "$nip" = "10.10.1.43" ]; then + echo -en "Mounting drive on '$nip'... ${idsCL[Default]}" + mntpath="/mnt/Nextcloud-Data" + ${NCMD} "test -e ${mntpath}/mounted" + if [ $? -eq 0 ]; then + echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}" + else + echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}" + fi + fi + + elif [ "${3}" != "q" ]; then + echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" + fi + + done + + fi + + + else + if [ "${4}" != "all" ]; then + if [ "${RUN_NODE_TYPE}" != "" ]; then + NT_HOSTS=${RUN_NODE_TYPE}_HOSTS[@] + NT_HOST=${RUN_NODE_TYPE}_HOSTS[0] + NTS=("${RUN_NODE_TYPE}"); + else + NTS=(${NM_NODE_TYPES[*]}) + fi + + else + NTS=(${NM_NODE_TYPES[*]}) + fi + + # if [ "${RUN_NODE_TYPE}" == "" ]; then + # + # fi + + if [ "${3}" != "q" ]; then + if [ "${NM_SERVICE_DESC[${1}]}" = "" ]; then + echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}" + exit 1 + fi + if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then + echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}" + exit 1 + fi + if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then + TACT='restart' + else + TACT="${2}" + fi + + echo + echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICE_DESC[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}" + echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" + fi + + + if [ "${1}" = "nginx" ] && ([ "${3}" == "" ] || [ "${3}" == "all" ]); then + [ "${RUN_NODE_TYPE}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && + + if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then + if [ "${3}" != "ns" ]; then + if [ "${3}" != "q" ]; then + echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}" + fi + if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then + PH_CMD="" + else + PH_CMD="ssh root@${!NT_HOST}" + fi + ${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl + # ${PH_CMD} "echo -e \"Service ${1} ${2}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl" + for nip in "${!NT_HOSTS}"; do + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then + checked=false + until [ "${checked}" = "true" ]; do + checked="`${PH_CMD} ssh root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl)) -le 90 ] && echo true`" + done + fi + done + ${PH_CMD} rm -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl + fi + fi + + elif [ "${1}" = "apache2-x" ]; then + if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then + if [ "${3}" != "ns" ]; then + if [ "${3}" != "q" ]; then + echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}" + fi + if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then + PH_CMD="ssh root@localhost" + else + PH_CMD="ssh root@${!NT_HOST}" + fi + echo "HERE: '$PH_CMD'" + + ${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl" + for nip in "${!NT_HOSTS}"; do + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + if [ "${nip}" != "${!NT_HOST}" ]; then + checked=false + until [ "${checked}" = "" ]; do + checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`" + done + fi + fi + done + ${PH_CMD} rm -f /etc/apache2/test.repl + fi + fi + fi + if [ "${3}" != "q" ]; then + echo -e "${idsCL[Green]}Completed${idsCL[Default]}" + echo + fi + + for NTYPE in "${NTS[@]}"; do + var1=${NTYPE}_SERVICES_CHECK[@] + # if [[ "${!var1}" = *"${1}"* ]]; then + if [[ " ${!var1} " =~ " ${1} " ]]; then + nid=1 + if [ "${3}" != "" ]; then + var2=(${3}) + sethost=true; + else + var2=${NTYPE}_HOSTS[@] + IFS=' ' + var2=(${!var2}) + unset IFS + sethost=false + echo $var2 + fi + + for nip in "${var2[@]}"; do + + # if [[ "${RUN_NODE_IP}" == *"${3}"* ]]; then GO=true; + # elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true; + # else GO=false; + # fi + # if [ "${GO}" == "true" ]; then + if [ $sethost == true ]; then + nodename=${NM_HOSTNAMES[${3}]} + else + nodename="${NM_HOSTNAMES[${nip}]}[${nip}]" + fi + if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then + nip='localhost ' + NCMD='' + else + NCMD="ssh root@${nip}" + fi + + NOGOCHK=true; + # if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then + if [ "${1}" == "gitea" ]; then + if [[ $($NCMD ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) != *10.10.1.120* ]]; then + NOGOCHK=false; + fi + fi + + if [ "${NOGOCHK}" == "true" ]; then + if [ "${3}" != "q" ]; then + echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}" + fi + checkhost=$(CHECK_HOST ${nip}) + if [ "${checkhost}" != "false" ]; then + [ "${1}" = "pdnsadmin" ] && srva="pdnsadmin.socket pdnsadmin.service" || srva=${1} + $NCMD systemctl ${2} ${srva} >/dev/null 2>&1 + if [ "${3}" != "q" ]; then + if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then + echo -e "${idsCL[Green]}Done${idsCL[Default]}" + elif [ "$2" = "stop" ]; then + echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}" + else + echo -e "${idsCL[Red]}ERROR${idsCL[Default]}" + fi + fi + elif [ "${3}" != "q" ]; then + echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" + fi + fi + # fi + nid=`expr $nid + 1` + # if [ "${1}" = "gitea" ]; then + # break + # fi + done + fi + done + fi + + if [ "${3}" != "q" ]; then + echo + echo -e "${idsCL[Green]}${NM_SERVICE_DESC[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}" + echo + fi +} + +CHECK_DOCKER_SERVICES(){ + + for NTYPE in "${NM_NODE_TYPES[@]}"; do + dockers=${NTYPE}_DOCKERS_CHECK[@] + hosts=${NTYPE}_HOSTS[@] + + if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsST[Bold]}"; DIVIDER + echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}" + DIVIDER; echo -e "${idsST[Reset]}" + fi + nid=1 + + for nip in "${!hosts}"; do + if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost' + else NCMD="ssh root@${nip}"; LH='' + fi + if [ "${NCMD}" != "" ]; then + checkhost=$(CHECK_HOST ${nip}) + fi + if [ "${checkhost}" != "false" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" + uptime=`${NCMD} uptime -p` + echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" + DIVIDER false green + fi + if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then + if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) + else + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) + fi + rm -f ${NM_TMPFOLDER}/${nip}.down + rm -f ${NM_TMPFOLDER}/${nip}.errtime + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" + fi + + for docker in "${!dockers}"; do + if [ "${ST_ACTION}" != "check" ]; then + c=0; cw=30; spc='' + spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` + until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done + echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: " + fi + if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then + ${NCMD} docker rm ${docker} + fi + ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d + sleep 10s + if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" + fi + fi + else + if [ "${ST_ACTION}" != "check" ]; then + echo -e "${idsCL[Green]}Running${idsCL[Default]}" + fi + fi + done + else + if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then + touch ${NM_TMPFOLDER}/${nip}.down + if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then + touch ${NM_TMPFOLDER}/${nip}.errtime + fi + else + errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) + fi + if [ "${ST_ACTION}" != "check" ]; then + toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) + echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" + fi + fi + if [ "${ST_ACTION}" != "check" ]; then echo; fi + nid=`expr $nid + 1` + done + fi + done + +} + + + diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 01315ff7..27c4a40f 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -11,95 +11,10 @@ source /opt/idssys/nodemgmt/defaults.inc source /opt/idssys/nodemgmt/inc/status.inc source /opt/idssys/nodemgmt/inc/certs.inc source /opt/idssys/nodemgmt/inc/sites.inc +source /opt/idssys/nodemgmt/inc/services.inc -CHECK_DOCKER_SERVICES(){ - - for NTYPE in "${NM_NODE_TYPES[@]}"; do - dockers=${NTYPE}_DOCKERS_CHECK[@] - hosts=${NTYPE}_HOSTS[@] - - if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsST[Bold]}"; DIVIDER - echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}" - DIVIDER; echo -e "${idsST[Reset]}" - fi - nid=1 - - for nip in "${!hosts}"; do - if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh root@${nip}"; LH='' - fi - if [ "${NCMD}" != "" ]; then - checkhost=$(CHECK_HOST ${nip}) - fi - if [ "${checkhost}" != "false" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}" - uptime=`${NCMD} uptime -p` - echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" - DIVIDER false green - fi - if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then - if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) - else - toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) - fi - rm -f ${NM_TMPFOLDER}/${nip}.down - rm -f ${NM_TMPFOLDER}/${nip}.errtime - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})" - fi - for docker in "${!dockers}"; do - if [ "${ST_ACTION}" != "check" ]; then - c=0; cw=30; spc='' - spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}` - until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done - echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: " - fi - if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then - ${NCMD} docker rm ${docker} - fi - ${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d - sleep 10s - if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}" - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}" - fi - fi - else - if [ "${ST_ACTION}" != "check" ]; then - echo -e "${idsCL[Green]}Running${idsCL[Default]}" - fi - fi - done - else - if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then - touch ${NM_TMPFOLDER}/${nip}.down - if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then - touch ${NM_TMPFOLDER}/${nip}.errtime - fi - else - errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down) - fi - if [ "${ST_ACTION}" != "check" ]; then - toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime) - echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" - fi - fi - if [ "${ST_ACTION}" != "check" ]; then echo; fi - nid=`expr $nid + 1` - done - fi - done -} BACKUP(){ BACKUP_FOLDER=/opt/idssys/backups/node-backup/$(date +%m-%d-%y)/$(date +%H-%M-%S) @@ -282,282 +197,6 @@ SET-PERMISSIONS(){ fi } -SERVICE(){ - #var=NC_HOSTS[@] - - if [ "$1" = "glusterd" ]; then - - for nip in "${GL_HOSTS[@]}"; do - if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then - nip='localhost ' - NCMD='' - else - NCMD="ssh root@${nip}" - fi - - if [ "${3}" != "q" ]; then - echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}" - fi - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - #$NCMD systemctl $2 $1 > /dev/null 2>&1 - if [ "${3}" != "q" ]; then - if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - elif [ "$2" = "stop" ]; then - echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}" - else - echo -e "${idsCL[Red]}ERROR${idsCL[Default]}" - fi - fi - elif [ "${3}" != "q" ]; then - echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" - fi - - done - - if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then - - echo - sleep 5s - - for nip in "${GL_HOSTS[@]}"; do - if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then - nip='localhost ' - NCMD='' - else - NCMD="ssh root@${nip}" - fi - - if [ "${3}" != "q" ]; then - echo -en "Mounting drive on '$nip'... ${idsCL[Default]}" - fi - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - #$NCMD mount -a > /dev/null 2>&1 - if [ "${3}" != "q" ]; then - # if [[ "${NCHOSTS}" == *"${nip}"* ]]; then - if [[ " ${NC_HOSTS[*]} " =~ " ${nip} " ]]; then - mntpath="/mnt/Nextcloud-Data" - # elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then - elif [[ " ${WEB_HOSTS[*]} " =~ " ${nip} " ]]; then - mntpath="/mnt/web-data" - fi - ${NCMD} "test -e ${mntpath}/mounted" - if [ $? -eq 0 ]; then - echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}" - else - echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}" - fi - fi - - if [ "$nip" = "10.10.1.43" ]; then - echo -en "Mounting drive on '$nip'... ${idsCL[Default]}" - mntpath="/mnt/Nextcloud-Data" - ${NCMD} "test -e ${mntpath}/mounted" - if [ $? -eq 0 ]; then - echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}" - else - echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}" - fi - fi - - elif [ "${3}" != "q" ]; then - echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" - fi - - done - - fi - - - else - if [ "${4}" != "all" ]; then - if [ "${RUN_NODE_TYPE}" != "" ]; then - NT_HOSTS=${RUN_NODE_TYPE}_HOSTS[@] - NT_HOST=${RUN_NODE_TYPE}_HOSTS[0] - NTS=("${RUN_NODE_TYPE}"); - else - NTS=(${NM_NODE_TYPES[*]}) - fi - - else - NTS=(${NM_NODE_TYPES[*]}) - fi - - # if [ "${RUN_NODE_TYPE}" == "" ]; then - # - # fi - - if [ "${3}" != "q" ]; then - if [ "${NM_SERVICE_DESC[${1}]}" = "" ]; then - echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}" - exit 1 - fi - if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then - echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}" - exit 1 - fi - if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then - TACT='restart' - else - TACT="${2}" - fi - - echo - echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICE_DESC[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}" - echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}" - fi - - - if [ "${1}" = "nginx" ] && ([ "${3}" == "" ] || [ "${3}" == "all" ]); then - [ "${RUN_NODE_TYPE}" != "" ] && [ "${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}" != "" ] && - - if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then - if [ "${3}" != "ns" ]; then - if [ "${3}" != "q" ]; then - echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}" - fi - if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then - PH_CMD="" - else - PH_CMD="ssh root@${!NT_HOST}" - fi - ${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl - # ${PH_CMD} "echo -e \"Service ${1} ${2}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl" - for nip in "${!NT_HOSTS}"; do - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then - checked=false - until [ "${checked}" = "true" ]; do - checked="`${PH_CMD} ssh root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl)) -le 90 ] && echo true`" - done - fi - done - ${PH_CMD} rm -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/test.repl - fi - fi - - elif [ "${1}" = "apache2-x" ]; then - if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then - if [ "${3}" != "ns" ]; then - if [ "${3}" != "q" ]; then - echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}" - fi - if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then - PH_CMD="ssh root@localhost" - else - PH_CMD="ssh root@${!NT_HOST}" - fi - echo "HERE: '$PH_CMD'" - - ${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl" - for nip in "${!NT_HOSTS}"; do - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - if [ "${nip}" != "${!NT_HOST}" ]; then - checked=false - until [ "${checked}" = "" ]; do - checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`" - done - fi - fi - done - ${PH_CMD} rm -f /etc/apache2/test.repl - fi - fi - fi - if [ "${3}" != "q" ]; then - echo -e "${idsCL[Green]}Completed${idsCL[Default]}" - echo - fi - - for NTYPE in "${NTS[@]}"; do - var1=${NTYPE}_SERVICES_CHECK[@] - # if [[ "${!var1}" = *"${1}"* ]]; then - if [[ " ${!var1} " =~ " ${1} " ]]; then - nid=1 - if [ "${3}" != "" ]; then - var2=(${3}) - sethost=true; - else - var2=${NTYPE}_HOSTS[@] - IFS=' ' - var2=(${!var2}) - unset IFS - sethost=false - echo $var2 - fi - - for nip in "${var2[@]}"; do - - # if [[ "${RUN_NODE_IP}" == *"${3}"* ]]; then GO=true; - # elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true; - # else GO=false; - # fi - # if [ "${GO}" == "true" ]; then - if [ $sethost == true ]; then - nodename=${NM_HOSTNAMES[${3}]} - else - nodename="${NM_HOSTNAMES[${nip}]}[${nip}]" - fi - if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then - nip='localhost ' - NCMD='' - else - NCMD="ssh root@${nip}" - fi - - NOGOCHK=true; - # if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then - if [ "${1}" == "gitea" ]; then - if [[ $($NCMD ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) != *10.10.1.120* ]]; then - NOGOCHK=false; - fi - fi - - if [ "${NOGOCHK}" == "true" ]; then - if [ "${3}" != "q" ]; then - echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}" - fi - checkhost=$(CHECK_HOST ${nip}) - if [ "${checkhost}" != "false" ]; then - [ "${1}" = "pdnsadmin" ] && srva="pdnsadmin.socket pdnsadmin.service" || srva=${1} - $NCMD systemctl ${2} ${srva} >/dev/null 2>&1 - if [ "${3}" != "q" ]; then - if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then - echo -e "${idsCL[Green]}Done${idsCL[Default]}" - elif [ "$2" = "stop" ]; then - echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}" - else - echo -e "${idsCL[Red]}ERROR${idsCL[Default]}" - fi - fi - elif [ "${3}" != "q" ]; then - echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}" - fi - fi - # fi - nid=`expr $nid + 1` - # if [ "${1}" = "gitea" ]; then - # break - # fi - done - fi - done - fi - - if [ "${3}" != "q" ]; then - echo - echo -e "${idsCL[Green]}${NM_SERVICE_DESC[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}" - echo - fi -} - HAPROXY-CONFIG() { echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}" DIVIDER false yellow @@ -628,67 +267,6 @@ HAPROXY-CONFIG() { echo } -SINGLE_SERVER_SERVICE_CHECK(){ - servicestocheck=${1^^}_SINGLESRVR_SERVICES[@] - dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@] - hosts=${1^^}_HOSTS[@] - - if [ "${!servicestocheck}" != "" ] || [ "${!dockerstocheck}" != "" ]; then - local_ips=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1); - local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1) - if [[ ${local_ips} = *10.10.1.120* ]]; then - if [ ! -f /mnt/web-data/mounted ]; then - /bin/mount -a - sleep 5s - fi - if [ -f ${NM_REPL_CHECK_LOC["${1}-data"]}/mounted ]; then - for stc in "${!servicestocheck}"; do - if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then - SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes" - echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE} - - [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" - for nip in "${!hosts}"; do - if [[ "${local_ip}" != *"${nip}"* ]]; then - ssh root@${nip} /bin/systemctl stop ${stc} - fi - done - sleep 3s - /bin/systemctl start ${stc} & - fi - done - for docker in "${!dockerstocheck}"; do - for nip in "${!hosts}"; do - if [[ "${local_ip}" = *"${nip}"* ]]; then - [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1 - [ "${docker}" == "authelia" ] && [ ! "$(/usr/bin/docker ps -q -f name=${docker}_redis)" ] && /usr/bin/docker start ${docker}_redis & >/dev/null 2>&1 - # [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "starting local" || echo "already started local" - else - [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && ssh root@${nip} /usr/bin/docker stop ${docker} & >/dev/null 2>&1 - [ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker}_redis)" ] && ssh root@${nip} /usr/bin/docker stop ${docker}_redis & >/dev/null 2>&1 - # [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping on $nip" || echo "already stopped on $nip" - fi - done - done - fi - - - else - for stc in "${!servicestocheck}"; do - [ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service" - /bin/systemctl stop ${stc} & - done - for docker in "${!dockerstocheck}"; do - [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker stop ${docker} & >/dev/null 2>&1 - # [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping" || echo "already stopped" - done - fi - - if [ "$(systemctl is-active nginx)" != "active" ]; then - ${NM_SCRIPT} service web restart ${local_ip} - fi - fi -} KEEPALIVE-CONFIG() { echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"