1378 lines
50 KiB
Bash
Executable File
1378 lines
50 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# NodeMgmt - Galera/NGINX Node Management Scripts
|
|
|
|
action="$1"
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/nodemgmt/settings.conf
|
|
source /opt/idssys/nodemgmt/defaults.inc
|
|
|
|
source /opt/idssys/nodemgmt/inc/status.inc
|
|
source /opt/idssys/nodemgmt/inc/certs.inc
|
|
source /opt/idssys/nodemgmt/inc/sites.inc
|
|
|
|
CHECK_DOCKER_SERVICES(){
|
|
|
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
|
dockers=${NTYPE}_DOCKER[@]
|
|
hosts=${NTYPE}_HOSTS[@]
|
|
|
|
if [[ -v ${NTYPE}_DOCKER ]]; then
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
echo -e "${idsST[Bold]}"; DIVIDER
|
|
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
|
|
DIVIDER; echo -e "${idsST[Reset]}"
|
|
fi
|
|
nid=1
|
|
|
|
for nip in "${!hosts}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
|
else NCMD="ssh root@${nip}"; LH=''
|
|
fi
|
|
if [ "${NCMD}" != "" ]; then
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
fi
|
|
if [ "${checkhost}" != "false" ]; then
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}"
|
|
uptime=`${NCMD} uptime -p`
|
|
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
|
DIVIDER false green
|
|
fi
|
|
if [ -f ${TMPFOLDER}/${nip}.down ]; then
|
|
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
|
|
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
|
|
else
|
|
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
|
|
fi
|
|
rm -f ${TMPFOLDER}/${nip}.down
|
|
rm -f ${TMPFOLDER}/${nip}.errtime
|
|
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}-UP" "${NM_HOSTNAMES[${nip}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
|
|
fi
|
|
|
|
for docker in "${!dockers}"; do
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
c=0; cw=30; spc=''
|
|
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
|
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
|
|
fi
|
|
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
|
|
${NCMD} docker rm ${docker}
|
|
fi
|
|
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d
|
|
sleep 10s
|
|
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
|
fi
|
|
else
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
else
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
done
|
|
else
|
|
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
|
|
touch ${TMPFOLDER}/${nip}.down
|
|
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
|
|
touch ${TMPFOLDER}/${nip}.errtime
|
|
fi
|
|
else
|
|
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
|
|
fi
|
|
if [ "${ST_ACTION}" != "check" ]; then
|
|
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
|
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
|
nid=`expr $nid + 1`
|
|
done
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
BACKUP(){
|
|
BACKUP_FOLDER=/opt/idssys/backups/node-backup/$(date +%m-%d-%y)/$(date +%H-%M-%S)
|
|
#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
|
|
#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
|
|
# if [ "${action}" != "backup" ]; then
|
|
echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
|
|
DIVIDER true
|
|
# fi
|
|
if [ ! -d ${BACKUP_FOLDER} ]; then
|
|
mkdir -p ${BACKUP_FOLDER}
|
|
fi
|
|
for item in "${!BACKUP_ITEMS[@]}"; do
|
|
c=0; spc=''; spc1=`expr 20 - ${#item}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en "Backing up '${item}'${spc}"
|
|
if [ -d ${BACKUP_ITEMS[$item]} ]; then
|
|
tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' >/dev/null 2>&1
|
|
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
|
|
fi
|
|
done
|
|
touch ${FOLDER}/backup.lastrun
|
|
# if [ "${action}" != "backup" ]; then
|
|
echo
|
|
DIVIDER true
|
|
# fi
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
ENTER2CONTINUE
|
|
fi
|
|
}
|
|
|
|
|
|
|
|
SET-PERMISSIONS(){
|
|
if [ "${1}" = "gitea" ]; then
|
|
nid=1
|
|
for nip in "${WEB_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
if [ "${2}" != "q" ]; then
|
|
echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
${NCMD} chown -R git:git /var/lib/gitea
|
|
${NCMD} chmod -R 750 /var/lib/gitea
|
|
${NCMD} chown -R git:git /home/git
|
|
${NCMD} chmod -R 750 /home/git
|
|
if [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
fi
|
|
elif [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
|
|
fi
|
|
nid=`expr $nid + 1`
|
|
done
|
|
elif [ "${1}" = "pdnsadmin" ]; then
|
|
nid=1
|
|
for nip in "${WEB_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
if [ "${2}" != "q" ]; then
|
|
echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
${NCMD} chown -R www-data:www-data /opt/powerdns-admin
|
|
${NCMD} chmod -R 775 /opt/powerdns-admin
|
|
if [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
fi
|
|
elif [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
|
|
fi
|
|
nid=`expr $nid + 1`
|
|
done
|
|
elif [ "${1}" = "nextcloud" ]; then
|
|
nip=${NC_HOSTS[0]}
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
if [ "${2}" != "q" ]; then
|
|
echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on Nextcloud-Node1 (${nip})... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
${NCMD} chown -R www-data:www-data /mnt/Nextcloud-Data
|
|
${NCMD} find /mnt/Nextcloud-Data/ -type d -exec chmod 770 {} \;
|
|
${NCMD} find /mnt/Nextcloud-Data/ -type f -exec chmod 660 {} \;
|
|
if [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
fi
|
|
elif [ "${2}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
|
|
fi
|
|
else
|
|
DIVIDER
|
|
if [ "${1}" != "" ]; then
|
|
NEW_SITE=${1}
|
|
echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}"
|
|
fi
|
|
if [ "${NEW_SITE}" = "xxx" ]; then
|
|
nid=1
|
|
for nip in "${MYSQL_HOSTS[@]}"; do
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -e "${idsCL[Yellow]}Setting folder permissions for MySQL-Node${nid} (${nip})${idsCL[Default]}"
|
|
echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}"
|
|
${NCMD} chown -R root:letsencrypt /etc/letsencrypt
|
|
${NCMD} chmod -R 6775 /etc/letsencrypt
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
nid=`expr $nid + 1`
|
|
done
|
|
fi
|
|
nid=1
|
|
for nip in "${WEB_HOSTS[@]}"; do
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
if [ "${NEW_SITE}" != "" ]; then
|
|
echo -en "${idsCL[Cyan]}Webserver-Node${nid} (${nip}) ${idsCL[Default]}"
|
|
${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE
|
|
${NCMD} chmod -R 7775 /var/www/$NEW_SITE
|
|
${NCMD} chown -R www-data:www-data /var/lib/php/sessions
|
|
${NCMD} chmod -R 7777 /var/lib/php/sessions
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}Setting folder permissions for Webserver-Node${nid} (${nip})${idsCL[Default]}"
|
|
echo -en "${idsCL[Cyan]}Webserver folders ${idsCL[Default]}"
|
|
${NCMD} chown -R www-data:www-data /var/www
|
|
${NCMD} chmod -R 7775 /var/www
|
|
${NCMD} chown -R www-data:www-data /var/lib/php/sessions
|
|
${NCMD} chmod -R 7777 /var/lib/php/sessions
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
echo -en "${idsCL[Cyan]}Gitea Folder ${idsCL[Default]}"
|
|
${NCMD} chown -R git:git /var/lib/gitea
|
|
${NCMD} chmod -R 750 /var/lib/gitea
|
|
${NCMD} chown -R git:git /home/git
|
|
${NCMD} chmod -R 750 /home/git
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}"
|
|
echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}"
|
|
${NCMD} chown -R root:letsencrypt /etc/letsencrypt
|
|
${NCMD} chmod -R 6775 /etc/letsencrypt
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
echo -en "${idsCL[Cyan]}PowerDNS-Admin ${idsCL[Default]}"
|
|
${NCMD} chown -R www-data:www-data /opt/powerdns-admin
|
|
${NCMD} chmod -R 775 /opt/powerdns-admin
|
|
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
fi
|
|
nid=`expr $nid + 1`
|
|
done
|
|
if [ "${NEW_SITE}" != "" ]; then echo ; fi
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
ENTER2CONTINUE
|
|
fi
|
|
fi
|
|
}
|
|
|
|
SERVICES(){
|
|
while :
|
|
do
|
|
echo
|
|
echo -e "${idsCL[Green]}Node Services${idsCL[Default]}"
|
|
DIVIDER true
|
|
s=1
|
|
for srvc in "${NODE_SERVICES[@]}"
|
|
do
|
|
echo " [${s}] ${NM_SERVICES[${srvc}]}"
|
|
s=`expr $s + 1`
|
|
done
|
|
echo
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
echo " [B] Back"
|
|
fi
|
|
echo " [Q] Quit"
|
|
echo
|
|
echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} "
|
|
read -n 1 selsrvc
|
|
selsrvc=`expr $selsrvc - 1`
|
|
echo
|
|
if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then
|
|
echo "Thats an invaild option,"
|
|
echo "please select a valid option only."
|
|
sleep 1
|
|
SERVICES
|
|
exit 0
|
|
elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
|
|
exit 0
|
|
elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
|
|
GUI
|
|
else
|
|
echo
|
|
echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}"
|
|
DIVIDER true
|
|
s=1
|
|
for srvcopt in "${NM_SRVCOPT[@]}"; do
|
|
echo " [${s}] ${srvcopt~}"
|
|
s=`expr $s + 1`
|
|
done
|
|
echo
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
echo " [B] Back"
|
|
fi
|
|
echo " [Q] Quit"
|
|
echo
|
|
echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} "
|
|
read -n 1 selsrvcopt
|
|
selsrvcopt=`expr $selsrvcopt - 1`
|
|
echo
|
|
if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${selsrvcopt}" != "B" ] && [ "${selsrvcopt}" != "b" ]; then
|
|
echo "Thats an invaild option,"
|
|
echo "please select a valid option only."
|
|
sleep 1
|
|
SERVICES
|
|
exit 0
|
|
elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
|
|
exit 0
|
|
elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
|
|
SERVICES
|
|
exit 0
|
|
else
|
|
echo
|
|
SERVICE ${NODE_SERVICES[${selsrvc}]} ${NM_SRVCOPT[${selsrvcopt}]}
|
|
echo
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
DIVIDER
|
|
ENTER2CONTINUE
|
|
fi
|
|
break
|
|
fi
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
SERVICE(){
|
|
TSI=$(/sbin/ip link | grep tailscale0)
|
|
MIP=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1)
|
|
[ ${#TSI} != 0 ] && TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1) || TSIP=""
|
|
NODE_TYPE=''
|
|
#var=NC_HOSTS[@]
|
|
|
|
if [ "$1" = "glusterd" ]; then
|
|
|
|
for nip in "${GL_HOSTS[@]}"; do
|
|
if [[ "${MIP}" == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
|
|
if [ "${3}" != "q" ]; then
|
|
echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
#$NCMD systemctl $2 $1 > /dev/null 2>&1
|
|
if [ "${3}" != "q" ]; then
|
|
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
|
|
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
|
|
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
|
|
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
|
|
elif [ "$2" = "stop" ]; then
|
|
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
elif [ "${3}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
|
|
fi
|
|
|
|
done
|
|
|
|
if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then
|
|
|
|
echo
|
|
sleep 5s
|
|
|
|
for nip in "${GL_HOSTS[@]}"; do
|
|
if [[ "${MIP}" == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
|
|
if [ "${3}" != "q" ]; then
|
|
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
#$NCMD mount -a > /dev/null 2>&1
|
|
if [ "${3}" != "q" ]; then
|
|
if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
|
|
mntpath="/mnt/Nextcloud-Data"
|
|
elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
|
|
mntpath="/mnt/web-data"
|
|
fi
|
|
${NCMD} "test -e ${mntpath}/mounted"
|
|
if [ $? -eq 0 ]; then
|
|
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
|
|
if [ "$nip" = "10.10.1.43" ]; then
|
|
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
|
|
mntpath="/mnt/Nextcloud-Data"
|
|
${NCMD} "test -e ${mntpath}/mounted"
|
|
if [ $? -eq 0 ]; then
|
|
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
|
|
elif [ "${3}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
|
|
fi
|
|
|
|
done
|
|
|
|
fi
|
|
|
|
|
|
else
|
|
if [ "${4}" != "all" ]; then
|
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
|
var=${NTYPE}_HOSTS[@]
|
|
NODE_TYPE=""
|
|
for nip in "${!var}"; do
|
|
if [[ "${MIP}" == *"${nip}"* ]]; then
|
|
NODE_TYPE=${NTYPE};
|
|
break 2
|
|
fi
|
|
done
|
|
done
|
|
|
|
if [ "${NODE_TYPE}" == "" ] && [ ${#TSI} != 0 ]; then
|
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
|
var=${NTYPE}_HOSTS[@]
|
|
for nip in "${!var}"; do
|
|
if [[ "${TSIP}" == *"${nip}"* ]]; then
|
|
NODE_TYPE=${NTYPE};
|
|
break 2
|
|
fi
|
|
done
|
|
done
|
|
fi
|
|
|
|
if [ "${NODE_TYPE}" != "" ]; then
|
|
NT_HOSTS=${NODE_TYPE}_HOSTS[@]
|
|
NT_HOST=${NODE_TYPE}_HOSTS[0]
|
|
NTS=("${NODE_TYPE}");
|
|
else
|
|
NTS=(${NODE_TYPES[*]})
|
|
fi
|
|
|
|
else
|
|
NTS=(${NODE_TYPES[*]})
|
|
fi
|
|
|
|
# if [ "${NODE_TYPE}" == "" ]; then
|
|
#
|
|
# fi
|
|
|
|
if [ "${3}" != "q" ]; then
|
|
if [ "${NM_SERVICES[${1}]}" = "" ]; then
|
|
echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
|
|
exit 1
|
|
fi
|
|
if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then
|
|
echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}"
|
|
exit 1
|
|
fi
|
|
if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then
|
|
TACT='restart'
|
|
else
|
|
TACT="${2}"
|
|
fi
|
|
|
|
echo
|
|
echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}"
|
|
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
|
|
fi
|
|
|
|
|
|
if [ "${1}" = "nginx" ]; then
|
|
[ "${NODE_TYPE}" != "" ] && [ "${NGINX_PATHS[${NODE_TYPE}]}" != "" ] &&
|
|
|
|
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
|
|
if [ "${3}" != "ns" ]; then
|
|
if [ "${3}" != "q" ]; then
|
|
echo -en "${idsCL[LightYellow]}Verifying replication across the nodes... ${idsCL[Default]}"
|
|
fi
|
|
if [[ "${MIP}" == *"${!NT_HOST}"* ]]; then
|
|
PH_CMD=""
|
|
else
|
|
PH_CMD="ssh root@${!NT_HOST}"
|
|
fi
|
|
${PH_CMD} touch ${NGINX_PATHS[${NODE_TYPE}]}/test.repl
|
|
# ${PH_CMD} "echo -e \"Service ${1} ${2}\" >> ${NGINX_PATHS[${NODE_TYPE}]}/test.repl"
|
|
for nip in "${!NT_HOSTS}"; do
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
|
|
checked=false
|
|
until [ "${checked}" = "true" ]; do
|
|
checked="`${PH_CMD} ssh root@${nip} [ -f ${NGINX_PATHS[${NODE_TYPE}]}/test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NGINX_PATHS[${NODE_TYPE}]}/test.repl)) -le 90 ] && echo true`"
|
|
done
|
|
fi
|
|
done
|
|
${PH_CMD} rm -f ${NGINX_PATHS[${NODE_TYPE}]}/test.repl
|
|
fi
|
|
fi
|
|
|
|
elif [ "${1}" = "apache2-x" ]; then
|
|
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
|
|
if [ "${3}" != "ns" ]; then
|
|
if [ "${3}" != "q" ]; then
|
|
echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}"
|
|
fi
|
|
if [[ "${MIP}" == *"${!NT_HOST}"* ]]; then
|
|
PH_CMD="ssh root@localhost"
|
|
else
|
|
PH_CMD="ssh root@${!NT_HOST}"
|
|
fi
|
|
echo "HERE: '$PH_CMD'"
|
|
|
|
${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl"
|
|
for nip in "${!NT_HOSTS}"; do
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
if [ "${nip}" != "${!NT_HOST}" ]; then
|
|
checked=false
|
|
until [ "${checked}" = "" ]; do
|
|
checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`"
|
|
done
|
|
fi
|
|
fi
|
|
done
|
|
${PH_CMD} rm -f /etc/apache2/test.repl
|
|
fi
|
|
fi
|
|
fi
|
|
if [ "${3}" != "q" ]; then
|
|
echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
|
|
for NTYPE in "${NTS[@]}"; do
|
|
var1=${NTYPE}SERVICES_CHECK
|
|
if [[ "${!var1}" = *"${1}"* ]]; then
|
|
nid=1
|
|
if [ "${3}" != "" ]; then
|
|
var2=(${3})
|
|
sethost=true;
|
|
else
|
|
var2=${NTYPE}_HOSTS[@]
|
|
IFS=' '
|
|
var2=(${!var2})
|
|
unset IFS
|
|
sethost=false
|
|
fi
|
|
|
|
for nip in "${var2[@]}"; do
|
|
|
|
|
|
# if [[ "${MIP}" == *"${3}"* ]]; then GO=true;
|
|
# elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true;
|
|
# else GO=false;
|
|
# fi
|
|
# if [ "${GO}" == "true" ]; then
|
|
if [ $sethost == true ]; then
|
|
nodename=${NM_HOSTNAMES[${3}]}
|
|
else
|
|
nodename="${NM_HOSTNAMES[${nip}]}"
|
|
fi
|
|
if [[ "${MIP}" == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
|
|
NOGOCHK=true;
|
|
# if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then
|
|
if [ "${1}" == "gitea" ]; then
|
|
if [[ $($NCMD ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) != *10.10.1.120* ]]; then
|
|
NOGOCHK=false;
|
|
fi
|
|
fi
|
|
|
|
if [ "${NOGOCHK}" == "true" ]; then
|
|
if [ "${3}" != "q" ]; then
|
|
echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
|
|
fi
|
|
checkhost=$(CHECK_HOST ${nip})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
[ "${1}" = "pdnsadmin" ] && srva="pdnsadmin.socket pdnsadmin.service" || srva=${1}
|
|
$NCMD systemctl ${2} ${srva} >/dev/null 2>&1
|
|
if [ "${3}" != "q" ]; then
|
|
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
|
|
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
|
|
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
|
|
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
|
|
elif [ "$2" = "stop" ]; then
|
|
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
elif [ "${3}" != "q" ]; then
|
|
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
|
|
fi
|
|
fi
|
|
# fi
|
|
nid=`expr $nid + 1`
|
|
# if [ "${1}" = "gitea" ]; then
|
|
# break
|
|
# fi
|
|
done
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ "${3}" != "q" ]; then
|
|
echo
|
|
echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
}
|
|
|
|
HAPROXY-CONFIG() {
|
|
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
|
|
DIVIDER false yellow
|
|
if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
|
|
nid=1
|
|
for nip in "${MYSQL_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
|
|
|
|
${NCMD} "nodemgmt update q"
|
|
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
|
|
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
|
|
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
echo
|
|
fi
|
|
if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
|
|
nid=1
|
|
for nip in "${WEB_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
|
|
|
|
${NCMD} "nodemgmt update q"
|
|
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
|
|
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
|
|
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
fi
|
|
if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
|
|
nid=1
|
|
for nip in "${LB_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -en "Creating File on LB-Node${nid} ($nip)... ${idsCL[Default]}"
|
|
|
|
${NCMD} "nodemgmt update q"
|
|
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
|
|
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
|
|
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
fi
|
|
SERVICE haproxy reload
|
|
echo
|
|
echo -e "${idsCL[Green]}HAProxy Config Files have been created.${idsCL[Default]}"
|
|
echo
|
|
}
|
|
|
|
SINGLE_SERVER_SERVICE_CHECK(){
|
|
servicestocheck=${1^^}_SINGLESRVR_SERVICES[@]
|
|
dockerstocheck=${1^^}_SINGLESRVR_DOCKERS[@]
|
|
hosts=${1^^}_HOSTS[@]
|
|
|
|
if [ "${!servicestocheck}" != "" ] || [ "${!dockerstocheck}" != "" ]; then
|
|
local_ips=$(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1);
|
|
local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1)
|
|
if [[ ${local_ips} = *10.10.1.120* ]]; then
|
|
if [ ! -f /mnt/web-data/mounted ]; then
|
|
/bin/mount -a
|
|
sleep 5s
|
|
fi
|
|
if [ -f ${REPL_CHECKS["${1}-data"]}/mounted ]; then
|
|
for stc in "${!servicestocheck}"; do
|
|
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
|
|
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
|
|
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
|
|
|
|
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
|
|
for nip in "${!hosts}"; do
|
|
if [[ "${local_ip}" != *"${nip}"* ]]; then
|
|
ssh root@${nip} /bin/systemctl stop ${stc}
|
|
fi
|
|
done
|
|
sleep 3s
|
|
/bin/systemctl start ${stc} &
|
|
fi
|
|
done
|
|
for docker in "${!dockerstocheck}"; do
|
|
for nip in "${!hosts}"; do
|
|
if [[ "${local_ip}" = *"${nip}"* ]]; then
|
|
[ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker start ${docker} & >/dev/null 2>&1
|
|
[ "${docker}" == "authelia" ] && [ ! "$(/usr/bin/docker ps -q -f name=${docker}_redis)" ] && /usr/bin/docker start ${docker}_redis & >/dev/null 2>&1
|
|
# [ ! "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "starting local" || echo "already started local"
|
|
else
|
|
[ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && ssh root@${nip} /usr/bin/docker stop ${docker} & >/dev/null 2>&1
|
|
[ "${docker}" == "authelia" ] && [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker}_redis)" ] && ssh root@${nip} /usr/bin/docker stop ${docker}_redis & >/dev/null 2>&1
|
|
# [ "$(ssh root@${nip} /usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping on $nip" || echo "already stopped on $nip"
|
|
fi
|
|
done
|
|
done
|
|
fi
|
|
|
|
|
|
else
|
|
for stc in "${!servicestocheck}"; do
|
|
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
|
|
/bin/systemctl stop ${stc} &
|
|
done
|
|
for docker in "${!dockerstocheck}"; do
|
|
[ "$(/usr/bin/docker ps -q -f name=${docker})" ] && /usr/bin/docker stop ${docker} & >/dev/null 2>&1
|
|
# [ "$(/usr/bin/docker ps -q -f name=${docker})" ] && echo "stopping" || echo "already stopped"
|
|
done
|
|
fi
|
|
|
|
if [ "$(systemctl is-active nginx)" != "active" ]; then
|
|
${SCRIPT} service web restart ${local_ip}
|
|
fi
|
|
fi
|
|
}
|
|
|
|
KEEPALIVE-CONFIG() {
|
|
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
|
|
DIVIDER false yellow
|
|
if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
|
|
nid=1
|
|
for nip in "${MYSQL_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
|
|
|
|
${NCMD} "nodemgmt update q"
|
|
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
|
|
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
|
|
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
echo
|
|
fi
|
|
if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
|
|
nid=1
|
|
for nip in "${WEB_HOSTS[@]}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
NCMD=''
|
|
else
|
|
NCMD="ssh root@${nip}"
|
|
fi
|
|
echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
|
|
|
|
${NCMD} "nodemgmt update q"
|
|
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
|
|
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
|
|
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
fi
|
|
SERVICE keepalived reload
|
|
echo
|
|
echo -e "${idsCL[Green]}Keepalive Config Files have been created.${idsCL[Default]}"
|
|
echo
|
|
}
|
|
|
|
NODEUPDATE() {
|
|
start=`date +%s`
|
|
if [ "${1}" = "report" ]; then
|
|
unset idsCL idsBG idsST
|
|
idsCL=('')
|
|
idsBG=('')
|
|
idsST=('')
|
|
fi
|
|
declare -i cw; declare -i spc1; declare -i c
|
|
echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
|
|
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
|
|
nip=${MYSQLMANAGER}
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost'
|
|
fi
|
|
c=0; cw=14; spca=''
|
|
spc1=${cw}-7
|
|
until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
|
|
c=0; cw=15; spcb=''
|
|
spc1=${cw}-${#nip}
|
|
until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
|
|
echo -en "Updating MySQL-Manager${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
|
|
ssh root@${nip} "nodemgmt update q"
|
|
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
|
|
ssh root@${nip} "runup runup"
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
echo
|
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
|
nid=1
|
|
var=${NTYPE}_HOSTS[@]
|
|
for nip in "${!var}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
|
|
nip='localhost '
|
|
fi
|
|
NCMD="ssh root@${nip}"
|
|
c=0; cw=14; spca=''
|
|
spc1=${cw}-${#NM_NODETYPES[$NTYPE]}
|
|
until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
|
|
c=0; cw=15; spcb=''
|
|
spc1=${cw}-${#nip}
|
|
until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
|
|
echo -en "Updating ${NM_HOSTNAMES[${nip}]}${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
|
|
${NCMD} "nodemgmt update q"
|
|
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
|
|
${NCMD} "runup runup"
|
|
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
|
|
nid=`expr $nid + 1`
|
|
done
|
|
echo
|
|
done
|
|
|
|
|
|
echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
|
|
echo
|
|
echo
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "runtime: ${runtime}"
|
|
}
|
|
|
|
VCENTER-SSL(){
|
|
VCHOSTNAME='vcenter.scity.us'
|
|
#BASECERTDIR="/etc/letsencrypt/live"
|
|
BASECERTDIR="/opt/nginx-proxy/ssl"
|
|
CERTDIR="${BASECERTDIR}/${VCHOSTNAME}"
|
|
SERVER="https://${VCHOSTNAME}"
|
|
CRED="administrator@scity.vs:$(pass iDSVC)"
|
|
VCUSER="administrator@scity.vs"
|
|
CURRENTLIVE=/etc/vmware-rhttpproxy/ssl/rui.crt
|
|
|
|
CERT=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.cer
|
|
KEY=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.key
|
|
CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
|
|
|
|
echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
|
|
VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
|
|
VCCERTDAYS=${VCCERTDAYS#*=}
|
|
|
|
if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
|
|
echo -e "${idsCL[Green]}${VCCERTDAYS} days left, Certificate is still valid, no noeed to update${idsCL[Default]}"
|
|
echo
|
|
exit 0
|
|
elif [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" == "force" ]; then
|
|
echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, forcing certificate update${idsCL[Default]}"
|
|
echo
|
|
else
|
|
echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, Certificate needs to be updated${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
|
|
LIVEMD5=$(ssh root@$VCHOSTNAME "md5sum $CURRENTLIVE | cut -d\ -f1")
|
|
CURRENTMD5=$(md5sum $CERTDIR/cert.pem | cut -d\ -f1)
|
|
if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
|
|
echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}"
|
|
echo
|
|
exit 0
|
|
fi
|
|
|
|
echo -e "${idsCL[Green]}Updating certificates on vCenter... ${idsCL[Default]}"
|
|
echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}"
|
|
echo
|
|
|
|
# eval $(awk '{ print "export " $1 }' /etc/sysconfig/vmware-environment)
|
|
|
|
scp $BASECERTDIR/$VCHOSTNAME/cert.pem root@${VCHOSTNAME}:$CERT
|
|
scp $BASECERTDIR/$VCHOSTNAME/privkey.pem root@${VCHOSTNAME}:$KEY
|
|
scp $BASECERTDIR/$VCHOSTNAME/fullchain.pem root@${VCHOSTNAME}:$CHAIN
|
|
|
|
ssh root@$VCHOSTNAME "(printf '1\n%s\n' '$VCUSER'; sleep 1; printf '%s\n' '$(pass iDSVC)'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '$CERT' '$KEY' '$CHAIN') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
|
|
|
|
|
|
|
|
# SESSION_ID=$(curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' --header 'vmware-use-header-authn: test' --header 'vmware-api-session-id: null' -u $CRED "$SERVER/rest/com/vmware/cis/session" --insecure | python3 -c "import sys, json; print(json.load(sys.stdin)['value'])")
|
|
# PRIVKEY=$(awk -v ORS='\\n' '1' "$CERTDIR/privkey.pem")
|
|
# CERT=$(awk -v ORS='\\n' '1' "$CERTDIR/fullcert.pem")
|
|
# REQUEST_BODY="{ \"spec\" : { \"cert\" : \"$CERT\", \"key\" : \"$PRIVKEY\" } }"
|
|
# curl --insecure -X PUT "$SERVER/rest/vcenter/certificate-management/vcenter/tls" \
|
|
# -H "vmware-api-session-id: $SESSION_ID" \
|
|
# -H "Content-type: application/json" \
|
|
# -d "$REQUEST_BODY"
|
|
# curl --insecure -X DELETE "$SERVER/rest/com/vmware/cis/session" -H "vmware-api-session-id: $SESSION_ID"
|
|
|
|
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
# echo
|
|
|
|
# read -n 1 -s -p "Press any key to continue"
|
|
|
|
# echo -en "${idsCL[LightCyan]}Stopping vCenter services... ${idsCL[Default]}"
|
|
# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --stop --all' >/dev/null 2>&1
|
|
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
#
|
|
# echo -en "${idsCL[LightCyan]}Starting vCenter services... ${idsCL[Default]}"
|
|
# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --start --all' >/dev/null 2>&1
|
|
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
|
|
|
|
echo
|
|
echo -e "${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}"
|
|
echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}"
|
|
echo
|
|
}
|
|
ADD_LOGROTATE_CRONTAB(){
|
|
if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then
|
|
(crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab -
|
|
fi
|
|
/bin/chmod 0644 ${FOLDER}/tmp-logrotate
|
|
echo
|
|
echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
|
|
echo
|
|
read -n 1 -s -p "Press any key to continue"
|
|
echo
|
|
}
|
|
|
|
UPDATE_DYNDNS(){
|
|
if [ "${1}" == "q" ]; then
|
|
SENDNOTICE "FOUND OLD DYNDNS SERVER" "IP=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1) - HOSTNAME=$(hostname)"
|
|
|
|
else
|
|
if [ "${2}" != 'q' ]; then
|
|
echo -e "${idsCL[LightCyan]}Checking for DYNDNS update at ${1^} ... ${idsCL[Default]}"
|
|
fi
|
|
|
|
if [ "${1}" == "home" ]; then
|
|
OLDIP=`dig +short sc1.scity.us @ns1.scity.us`
|
|
elif [ "${1}" == "offsite" ]; then
|
|
OLDIP=`dig +short offsite.scity.us @ns1.scity.us`
|
|
fi
|
|
NEWIP=`dig +short myip.opendns.com @resolver1.opendns.com`
|
|
|
|
if ([ "${OLDIP}" != "" ] && [ "${NEWIP}" != "" ] && [[ "${NEWIP}" != *";"* ]] && [ "${OLDIP}" != "${NEWIP}" ]) || [ "${2}" == "force" ]; then
|
|
if [ "${1}" == "home" ]; then
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=sc1.scity.us&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=sc2.scity.us&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=scity.us&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=schroedercity.com&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=dasrei.com&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1
|
|
|
|
elif [ "${1}" == "offsite" ]; then
|
|
/usr/bin/curl -s "https://systems:Dcs9613@wdns.scity.us/nic/update?hostname=offsite.scity.us&myip=${NEWIP}" > /dev/null 2>&1
|
|
|
|
fi
|
|
SENDNOTICE "Updated ${1^} WAN IP from '${OLDIP}' to '${NEWIP}'" "${1^^} DYNDNS Updated"
|
|
[ "${2}" != 'q' ] && echo -e "${idsCL[Green]}Updated ${1^} WAN IP from '${OLDIP}' to '${NEWIP}'${idsCL[Default]}"
|
|
|
|
elif [[ "${NEWIP}" == *";"* ]]; then
|
|
[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could reach any servers${idsCL[Default]}"
|
|
# SENDNOTICE "Connection error, could reach any servers" "${1^^} DYNDNS Could not Update"
|
|
elif [ "${OLDIP}" == "" ]; then
|
|
[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could not get OLD IP${idsCL[Default]}"
|
|
elif [ "${NEWIP}" == "" ]; then
|
|
[ "${2}" != 'q' ] && echo -e "${idsCL[Red]}Could not get NEW IP${idsCL[Default]}"
|
|
else
|
|
[ "${2}" != 'q' ] && echo -e "${idsCL[Green]}${1^^} WAN IP has not changed '${OLDIP}' = '${NEWIP}'${idsCL[Default]}"
|
|
fi
|
|
echo
|
|
|
|
# if [ "${2}" != 'q' ]; then
|
|
# read -n 1 -s -p "Press any key to continue"
|
|
# echo
|
|
# fi
|
|
fi
|
|
}
|
|
|
|
RUN_COMMAND(){
|
|
echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
|
|
for NTYPE in "${NODE_TYPES[@]}"; do
|
|
echo -e "${idsST[Bold]}"; DIVIDER
|
|
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Nodes to run command { \`${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}\` }${idsCL[Default]}"
|
|
DIVIDER; echo -e "${idsST[Reset]}"
|
|
nid=1
|
|
|
|
var=${NTYPE}_HOSTS[@]
|
|
for nip in "${!var}"; do
|
|
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
|
else NCMD="ssh root@${nip}"; LH=''
|
|
fi
|
|
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
|
|
DIVIDER false green
|
|
#echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
|
|
${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}
|
|
DIVIDER false green
|
|
echo
|
|
nid=`expr $nid + 1`
|
|
done
|
|
echo
|
|
done
|
|
}
|
|
|
|
DOWNLIST(){
|
|
if [ "${1}" = "report" ]; then
|
|
unset idsCL idsBG idsST
|
|
idsCL=('')
|
|
idsBG=('')
|
|
idsST=('')
|
|
fi
|
|
declare -i cw; declare -i spc1; declare -i c
|
|
DOWN_LIST=`find ${FOLDER} -name "*.down"`
|
|
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
DIVIDER true
|
|
fi
|
|
echo
|
|
|
|
if [ "${DOWN_LIST}" == "" ]; then
|
|
echo -e "${idsCL[LightCyan]}Nothing is down${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[LightGreen]}Current List of Items Down...${idsCL[Default]}"
|
|
DIVIDER false yellow 95
|
|
echo -e "Service/Host Host Down_Time"
|
|
DIVIDER false yellow 95
|
|
|
|
cw=50; cw2=65
|
|
for downitem in ${DOWN_LIST}; do
|
|
ditem=${downitem##*/}
|
|
ditem=${ditem//.down}
|
|
host=${ditem%%~*}
|
|
item=${ditem#*~}
|
|
|
|
if [[ $ditem == *"~"* ]] && [ "${NM_SERVICES[${item}]}" != "" ]; then
|
|
itemname=${NM_SERVICES[${item}]}
|
|
elif [[ $ditem == *"~"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then
|
|
itemname="${NM_DOCKERS[${item}]} (docker)"
|
|
elif [[ $ditem == *"~"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then
|
|
itemname="${REPL_DESC[${item}]} (replication)"
|
|
else
|
|
itemname='HOST DOWN';
|
|
[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"
|
|
fi
|
|
|
|
if [ -f ${TMPFOLDER}/${ditem}.errtime ]; then
|
|
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.errtime)
|
|
else
|
|
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.down)
|
|
fi
|
|
|
|
c=0; spc=''
|
|
spc1=${cw}-${#itemname}
|
|
#echo "HERE: $downitem ~ $ditem ~ $host ~ $item ~ $itemname ~ $spc1"
|
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
c=0; spc2=''
|
|
spc1=${cw2}-${#itemname}-${#host}-${spc1}
|
|
until [ $c = ${spc1} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
|
|
echo -e "${idsST[Bold]}${idsCL[Red]}${itemname}${spc}${host}${spc2}$(SHOW_TIME ${toterrtime})${idsST[Reset]}${idsCL[Default]}"
|
|
DIVIDER false darkGray 80
|
|
done
|
|
fi
|
|
echo
|
|
if [ -z $action ] || [ "${action}" = "gui" ]; then
|
|
DIVIDER true
|
|
ENTER2CONTINUE
|
|
fi
|
|
echo
|
|
}
|
|
|
|
UPDATEPUSHOVERINFO(){
|
|
echo
|
|
if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
|
|
echo -en "${idsCL[LightCyan]}Use current Pushover Info:\nApp Token: [${idsCL[White]}${PUSHOVER_APP_TOKEN}${idsCL[LightCyan]}\nUser Token: [${idsCL[White]}${PUSHOVER_USER_TOKEN}${idsCL[LightCyan]}] (Y/n)? ${idsCL[Default]}"
|
|
read -n 1 choice
|
|
case "$choice" in
|
|
[Nn]) PUSHOVER_APP_TOKEN="";PUSHOVER_USER_TOKEN="";;esac
|
|
fi
|
|
if [ "${PUSHOVER_APP_TOKEN}" == "" ]; then
|
|
echo; echo -e -n "${idsCL[LightCyan]}Pushover App Token: ${idsCL[Default]}"
|
|
read -e PUSHOVER_APP_TOKEN
|
|
echo; echo -e -n "${idsCL[LightCyan]}(optional) Pushover User Token: ${idsCL[Default]}"
|
|
read -e PUSHOVER_USER_TOKEN
|
|
fi
|
|
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
|
|
host_name[${hostid}]=$hostname
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
|
|
|
|
for hostid in "${!host_ip[@]}"; do
|
|
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running commands${idsCL[Default]}${idsST[Reset]}"
|
|
ssh root@${host_ip[$hostid]} "/bin/sed -i '/PUSHOVER_/d' /opt/idssys/nodemgmt/settings.conf"
|
|
if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
|
|
echo "putting in info 'root@${host_ip[$hostid]}'"
|
|
ssh root@${host_ip[$hostid]} "echo PUSHOVER_APP_TOKEN=\\\"${PUSHOVER_APP_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes1'"
|
|
else
|
|
echo "taking out info"
|
|
ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_APP_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
|
|
fi
|
|
if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then
|
|
ssh root@${host_ip[$hostid]} "echo PUSHOVER_USER_TOKEN=\\\"${PUSHOVER_USER_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes2'"
|
|
else
|
|
ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_USER_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
|
|
fi
|
|
|
|
else
|
|
echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
|
|
fi
|
|
done
|
|
|
|
}
|
|
|
|
|
|
GUI(){
|
|
DISP_HEADER true false
|
|
while :
|
|
do
|
|
echo " [1] Node Status"
|
|
echo " [2] Node Services"
|
|
echo
|
|
echo " [3] New Site"
|
|
echo " [4] Delete Site"
|
|
echo
|
|
echo " [5] New Certificate"
|
|
echo " [6] Renew Certificates"
|
|
echo " [7] List Certificates"
|
|
echo
|
|
echo " [8] Set folder permissions on nodes"
|
|
echo " [9] Backup Node Settings and Files"
|
|
echo
|
|
echo " [D] Update Primary DNS Records"
|
|
echo " [R] Add cron task to rotate wtmp & btmp log files"
|
|
echo
|
|
echo " [Q] Quit"
|
|
echo
|
|
echo
|
|
echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
|
|
read -n 1 opt
|
|
echo
|
|
case $opt in
|
|
1) STATUS; GUI;;
|
|
2) SERVICES; GUI;;
|
|
3) NEWSITE; GUI;;
|
|
4) DELSITES; GUI;;
|
|
5) NEWCERT; GUI;;
|
|
6) CERTRENEW; GUI;;
|
|
7) LISTCERTS; GUI;;
|
|
8) SET-PERMISSIONS; GUI;;
|
|
9) BACKUP; GUI;;
|
|
[Dd]) UPDATE_DYNDNS; GUI;;
|
|
[Rr]) ADD_LOGROTATE_CRONTAB; GUI;;
|
|
[Qq]) EXIT1
|
|
exit 0;;
|
|
*) echo "Thats an invaild option,";
|
|
echo "please select a valid option only.";
|
|
sleep 1;;
|
|
esac
|
|
done
|
|
}
|
|
# if [ ${action-x} ]; then
|
|
if [[ "${noheader}" != *" ${action} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "${2}" != "q" ] && [ "${3}" != "q" ] && [ "${1}" != "gui" ] && [ "${1}" != "nightlyrenew" ] && [ "${1}" != "update" ]; then
|
|
DISP_HEADER
|
|
fi
|
|
case $action in
|
|
newcert) NEWCERT ${2};;
|
|
certrenew) CERTRENEW;;
|
|
delssl) DEL-SSL ${2};;
|
|
listcerts) LISTCERTS;;
|
|
listcerts-npm) LISTCERTS_NPM;;
|
|
copynpmcerts) COPYCERTS_NPM ${2};;
|
|
checknpmcerts) CHECK_NPMCERTS;;
|
|
checkcerts) CHECK-CERTS ${2};;
|
|
nightlyrenew) NIGHTLYRENEW;;
|
|
delsite) DELSITE ${2} ${3} ${4} ${5};;
|
|
delsites) DELSITES;;
|
|
newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} ${23} ${24} ${25};;
|
|
siteinfo) SITEINFO ${2} ${3};;
|
|
|
|
update) ;;
|
|
update-nodes)
|
|
if [ "${2}" == "report" ]; then
|
|
rm -f ${LOGFOLDER}/update-nodes.lastrun
|
|
NODEUPDATE ${2} > ${LOGFOLDER}/update-nodes.lastrun
|
|
else
|
|
NODEUPDATE ${2}
|
|
fi
|
|
;;
|
|
update-dyndns) UPDATE_DYNDNS ${2} ${3};;
|
|
backup) BACKUP;;
|
|
downlist)
|
|
if [ "${2}" = "report" ]; then
|
|
if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then
|
|
rm -f ${LOGFOLDER}/status-downlist.lastrun
|
|
DOWNLIST report >> ${LOGFOLDER}/status-downlist.lastrun
|
|
STATUSLOG="$(cat ${LOGFOLDER}/status-downlist.lastrun)"
|
|
SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
|
|
fi
|
|
else
|
|
DOWNLIST ${2}
|
|
fi;;
|
|
status)
|
|
if [ -f "${FOLDER}/status-check.running" ]; then
|
|
fd=`stat --format=%Y "${FOLDER}/status-check.running"`
|
|
cbd=$((`date +%s` - 450))
|
|
[ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running"
|
|
fi
|
|
if [ ! -f "${FOLDER}/status-check.running" ]; then
|
|
start_status=`date +%s`
|
|
touch "${FOLDER}/status-check.running"
|
|
if [ "${2}" == "report" ]; then
|
|
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${LOGFILE}
|
|
echo -e "$(date) - Starting status check #############################\n" > ${FOLDER}/status-check.running
|
|
STATUS report ${3} >> ${FOLDER}/status-check.running
|
|
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${FOLDER}/status-check.running
|
|
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${LOGFILE}
|
|
rm -f ${FOLDER}/status-check.lastrun
|
|
mv ${FOLDER}/status-check.running ${FOLDER}/status-check.lastrun
|
|
end_status=`date +%s`
|
|
runtime_status=$((end_status-start_status))
|
|
echo -e "\nRuntime: ${runtime_status}" >> ${FOLDER}/status-check.lastrun
|
|
if [ "${3}" = "email" ]; then
|
|
echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
|
|
DOWNLIST report >> ${FOLDER}/status-check.lastrun
|
|
STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
|
|
SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
|
|
fi
|
|
else
|
|
STATUS ${2} ${3}
|
|
rm -f ${FOLDER}/status-check.running
|
|
fi
|
|
else
|
|
# echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
|
|
echo -e "${idsCL[LightYellow]}Status checks are already running${idsCL[Default]}"
|
|
echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}"
|
|
read -n1 choice
|
|
case $choice in
|
|
[Ll]) watch -n1 tail -n35 ${FOLDER}/status-check.running; echo;;
|
|
[Pp]) echo; cat ${FOLDER}/status-check.lastrun;;
|
|
esac
|
|
echo
|
|
fi;;
|
|
checkdocker) CHECK_DOCKER_SERVICES ${2};;
|
|
set-permissions) SET-PERMISSIONS ${2};;
|
|
service)
|
|
if [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "start" ]; then
|
|
#SET-PERMISSIONS ${2}
|
|
SERVICE ${2} start
|
|
elif [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "restart" ]; then
|
|
SERVICE ${2} stop
|
|
#SET-PERMISSIONS ${2}
|
|
SERVICE ${2} start
|
|
elif [ "$2" = "web" ]; then
|
|
SERVICE php5.6-fpm ${3} ${4}
|
|
SERVICE php7.2-fpm ${3} ${4}
|
|
SERVICE php7.3-fpm ${3} ${4}
|
|
SERVICE php7.4-fpm ${3} ${4}
|
|
SERVICE php8.2-fpm ${3} ${4}
|
|
SERVICE nginx ${3} ${4}
|
|
elif [ "$2" = "glusterdxx" ]; then
|
|
SERVICE glusterd stop
|
|
else
|
|
SERVICE ${2} ${3} ${4}
|
|
fi;;
|
|
services) SERVICES;;
|
|
singleservercheck) SINGLE_SERVER_SERVICE_CHECK ${2} ${3};;
|
|
haproxy-config) HAPROXY-CONFIG;;
|
|
keepalive-config) KEEPALIVE-CONFIG;;
|
|
vc-ssl) VCENTER-SSL ${2};;
|
|
run) RUN_COMMAND ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15};;
|
|
update-pushover) UPDATEPUSHOVERINFO;;
|
|
gui) GUI;;
|
|
*)
|
|
echo -e "Commands followed by '-h' will show addtional info. {work-in-progress}"
|
|
echo
|
|
echo -e "Usage: ${idsCL[LightCyan]}nodemgmt ${idsCL[Yellow]}{COMMAND}${idsCL[Default]} {"
|
|
echo -e " ${idsCL[Yellow]}backup${idsCL[Default]} - Run backup of node items"
|
|
echo -e " ${idsCL[Yellow]}certrenew${idsCL[Default]} - Renew all SSL Certificates"
|
|
echo -e " ${idsCL[Yellow]}checkcerts${idsCL[Default]} - Check SSL Expirations"
|
|
echo -e " ${idsCL[Yellow]}delsite${idsCL[Default]} - Delete entire site"
|
|
echo -e " ${idsCL[Yellow]}delsites${idsCL[Default]} - (UI) Delete entire site"
|
|
echo -e " ${idsCL[Yellow]}delssl${idsCL[Default]} - Delete SSL Certificate"
|
|
echo -e " ${idsCL[Yellow]}gui${idsCL[Default]} - Nodemgmt UI interface"
|
|
echo -e " ${idsCL[Yellow]}haproxy-config${idsCL[Default]} - Push out haproxy config"
|
|
echo -e " ${idsCL[Yellow]}listcerts${idsCL[Default]} - List all SSL Certificates and Common Names"
|
|
echo -e " ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)] - Generate new SSL Certificate"
|
|
echo -e " ${idsCL[Yellow]}newsite${idsCL[Default]} - Create a new site"
|
|
echo -e " ${idsCL[Yellow]}nightlyrenew${idsCL[Default]} - Renewal of SSL Certs, designed for cron tasks"
|
|
echo -e " ${idsCL[Yellow]}run${idsCL[Default]} - Run a command on all Nodes"
|
|
echo -e " ${idsCL[Yellow]}service${idsCL[Default]} [service] [action] - Manages services accross like nodes"
|
|
echo -e " ${idsCL[Yellow]}services${idsCL[Default]} - (UI) Manage services accross all nodes"
|
|
echo -e " ${idsCL[Yellow]}set-permissions${idsCL[Default]} - Set specific permissions preset for app folders"
|
|
echo -e " ${idsCL[Yellow]}siteinfo${idsCL[Default]} - View detailed nginx site info"
|
|
echo -e " ${idsCL[Yellow]}status${idsCL[Default]} - Check Status of systems"
|
|
echo -e " ${idsCL[Yellow]}update${idsCL[Default]} - Update nodemgmt"
|
|
echo -e " ${idsCL[Yellow]}update-nodes${idsCL[Default]} - Update nodemgmt on all nodes"
|
|
echo -e " ${idsCL[Yellow]}update-dyndns${idsCL[Default]} - Update DYNDNS records"
|
|
echo -e " ${idsCL[Yellow]}vc-ssl${idsCL[Default]} - Update vCenter SSL"
|
|
echo -e "}"
|
|
echo
|
|
echo
|
|
exit 0;;
|
|
esac
|
|
# else
|
|
# GUI
|
|
# fi
|
|
|
|
exit 0 |