voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity
Files
7ebc912d72ac6120c63c7cd142036b5cc41a04e0
NodeMgmt/nodemgmt-scripts.sh

2710 lines
96 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
# NodeMgmt - Galera/NGINX Node Management Scripts
action="$1"
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/nodemgmt/settings.conf
source /opt/idssys/nodemgmt/defaults.inc
#source /opt/idssys/nodemgmt/ssl-check.sh
# ========================================================= #
STATUS(){
start=`date +%s`
ST_ACTION=${1}
if [ "${ST_ACTION}" = "report" ]; then
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
elif [ "${ST_ACTION}" = "sync" ]; then
ST_ACTION=repl
fi
declare -i cw; declare -i spc1; declare -i c
########################
if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
if [ ! -f ${FOLDER}/test.repl ]; then
touch ${FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -en "${idsCL[LightCyan]}Setting up status checks... ${idsCL[Default]}"
fi
#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
#LBHCMD="ssh root@${LB_HOSTS[0]}"
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl
yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl
else
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl"
# ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl"
# yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl"
fi
done
fi
done
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi
[ "${ST_ACTION}" != "repl" ] && CHECKFOLDERSYNC=false || CHECKFOLDERSYNC=true
else
rm -Rf ${FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[LightCyan]}Will verify folder syncs... ${idsCL[Default]}"
CHECKFOLDERSYNC=true
fi
fi
fi
########################
if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "dockers" ] || [ "${ST_ACTION}" = "check" ]; then
if [ ! -z ${LOCAL_SERVICES+x} ]; then
if [ "${ST_ACTION}" != "check" ]; then
lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
uptime=`uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
DIVIDER false green
fi
for srvc in "${LOCAL_SERVICES[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${NM_SERVICES[${srvc}]}${spc}: "
fi
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then
if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down)
fi
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down
rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICES[${srvc}]} is back up" >> ${LOGFILE}
fi
fi
done
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi
########################
for NTYPE in "${NODE_TYPES[@]}"; do
nid=1
dockers=${NTYPE}_DOCKER[@]
hosts=${NTYPE}_HOSTS[@]
var=${NTYPE}_HOSTS[@]
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" = "dockers" ]; then
GOFORCHECK=false;
else
GOFORCHECK=true;
fi
if [ ${GOFORCHECK} = true ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
if [ "${ST_ACTION}" = "dockers" ]; then
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
fi
DIVIDER; echo -e "${idsST[Reset]}"
fi
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" = "report" ] && [ "${2}" = "email" ]; then
uptime=`${NCMD} uptime -p`
elif [ "${ST_ACTION}" != "report" ]; then
uptime=`${NCMD} uptime -p`
fi
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
if [ "${ST_ACTION}" != "dockers" ]; then
var2=${NTYPE}_SERVICES_CHECK[@]
for srvc in "${!var2}"; do
NOGOCHK=true;
# if [ ${srvc} = "gitea" ] || [ ${srvc} = "pdnsadmin" ] || [ ${srvc} = "pdnsadmin.socket" ]; then
if [ ${srvc} = "gitea" ]; then
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]] && [[ "${NTYPE}" != "LPD" ]]; then
NOGOCHK=false;
fi
fi
if [ ${NOGOCHK} = true ]; then
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=${cw}-${#NM_SERVICES[${srvc}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
fi
srvctst=$(${NCMD} systemctl is-active ${srvc})
#echo "${NCMD} systemctl is-active ${srvc} = ${srvctst}"
#if [ "${srvctst}" = "active" ]; then echo "yes"; else echo "no"; fi
if [ "${srvctst}" != "active" ] && [ "${srvc}" = "mysql" ]; then
mysqlgo=false;
[ "$(${NCMD} systemctl is-active mariadb)" = "active" ] && mysqlgo=true;
elif [ "${srvctst}" != "active" ] && [ "${srvc}" = "mariadb" ]; then
mysqlgo=false;
[ "$(${NCMD} systemctl is-active mysql)" = "active" ] && mysqlgo=true;
elif [ "${srvctst}" = "active" ] && ([ "${srvc}" = "mysql" ] || [ "${srvc}" = "mariadb" ]); then
mysqlgo=true
fi
# echo "mysqlgo=${mysqlgo}"
if [ "${srvctst}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is down" 1
touch ${FOLDER}/${nip}-${srvc}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
touch ${FOLDER}/${nip}-${srvc}.down
if [ "${srvc}" = "mysql" ]; then
${NCMD} systemctl restart ${srvc}
fi
fi
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then
if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down)
fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_SERVICES[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${LOGFILE}
rm -f ${FOLDER}/${nip}-${srvc}.down
rm -f ${FOLDER}/${nip}-${srvc}.errtime
fi
# if [ "${srvc}" == "pdnsadmin.socket" ] || [ "${srvc}" == "pdnsadmin" ] || [ "${srvc}" == "gitea" ]; then
if [ "${srvc}" == "gitea" ]; then
rm -f ${FOLDER}/*-${srvc}.down
rm -f ${FOLDER}/*-${srvc}.errtime
fi
fi
[ "${mysqlgo}" = "true" ] && unset mysqlgo
fi
done
fi
if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
echo -e "${idsCL[Yellow]}---------------------------------------------${idsCL[Default]}"
fi
for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker} >/dev/null 2>&1
fi
${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
rm -f ${FOLDER}/${nip}-${docker}.down
rm -f ${FOLDER}/${nip}-${docker}.errtime
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${FOLDER}/${nip}-${docker}.down ]; then
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_DOCKERS[${docker}]} is down" 1
touch ${FOLDER}/${nip}-${docker}.down
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - ${NM_DOCKERS[${docker}]} (docker) is down" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
if [ ! -f ${FOLDER}/${nip}-${docker}.errtime ]; then
mv ${FOLDER}/${nip}-${docker}.down ${FOLDER}/${nip}-${docker}.errtime
fi
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${docker}.errtime)
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
fi
touch ${FOLDER}/${nip}-${docker}.down
fi
fi
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
rm -f ${FOLDER}/${nip}-${docker}.down
rm -f ${FOLDER}/${nip}-${docker}.errtime
fi
done
fi
else
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node is down!${idsCL[Default]}"
fi
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - Node is down" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
# if [ $errtime -gt ${RENOTIFY} ] && [ "${EMAIL_NOTICE}" != "" ]; then
# echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${EMAIL_NOTICE}
# fi
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
fi #GOFORCHECK
done
fi
########################
## REPLICATION CHECK
########################
if [ "${ST_ACTION}" == "" ] || [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ]; then
if [ "${CHECKFOLDERSYNC}" == true ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
for NTYPE in "${NODE_TYPES[@]}"; do
#GOREPCHK=false;
#CHR=`date +"%H"`;
#if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then
# GOREPCHK=true;
#elif [ "${NTYPE}" != "NC" ]; then
# GOREPCHK=true;
#fi
GOREPCHK=true;
if [ "${GOREPCHK}" = "true" ]; then
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
nid=1
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${nip}"
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false green
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
var2=${NTYPE}_REPL_CHECK[@]
# timeout=`date --date='20 seconds' +%s`
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=${cw}-${#REPL_DESC[${rcheck}]}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
timeout=`date --date='2 minutes' +%s`
# if [ "${rcheck}" == "www" ]; then
# timeout=`date --date='2 minutes' +%s`
# else
# timeout=`date --date='20 seconds' +%s`
# fi
else
timeout=`date --date='2 minutes' +%s`
fi
warn_timeout=`date --date='30 secs' +%s`
checked=false
until [ "${checked}" = "" ]; do
if [ "${PH_CMD}" = "" ]; then
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
else
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
fi
if [ "`date +%s`" -gt "${timeout}" ]; then
timeout=true
break
elif [ "`date +%s`" -gt "${warn_timeout}" ]; then
echo -en " waiting for sync... "
warn_timeout=`date --date='100 minutes' +%s`
fi
done
if [ "${timeout}" = "true" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then
touch ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
fi
if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then
rm -f ${FOLDER}/${nip}-${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_NODETYPES[$NTYPE]}-Node${nid} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}"
fi
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
fi
nid=`expr $nid + 1`
done
fi
fi
done
##########################
# REMOVE REPL CHECK FILES
##########################
if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}"
fi
#NHCMD="ssh root@${MYSQL_HOSTS[0]}"
#LBHCMD="ssh root@${LB_HOSTS[0]}"
#WHCMD="ssh root@${WEB_HOSTS[0]}"
for NTYPE in "${NODE_TYPES[@]}"; do
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
else
${PH_CMD} rm -f ${FOLDER}/test.repl
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl
fi
done
fi
done
rm -Rf ${FOLDER}/test.repl
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo
fi
fi
fi
fi
########################
## FREE SPACE CHECK
########################
#if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then
if [ "${ST_ACTION}" = "freespace" ]; then
if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then
for NTYPE in "${NODE_TYPES[@]}"; do
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${ST_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
declare -A partitions
partitions=`${NCMD} df -hBM | grep -vE '^Filesystem|tmpfs|cdrom|loop|overlay|shm|-volume|Music|media-data|Software' | awk '{ print $5 " " $1 }' | while read output; do echo $output | awk '{ print $2 }'; done;`
partitions2=`echo ${partitions[*]}`
partitions3=(${partitions2})
for partition in "${partitions3[@]}"; do
if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
freespace=`${NCMD} df -hPBM ${partition} | awk '{print $4}' |tail -1|sed 's/M$//g'` >/dev/null 2>&1
if [ "${freespace}" -le "1000" ]; then
fs_status='error'
fs_status_color='Red'
SENDNOTICE "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free" 1
elif [ "${freespace}" -le "5000" ]; then
fs_status='warn'
fs_status_color='Yellow'
SENDNOTICE "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" "${partition} : ${freespace} MB free"
else
fs_status=''
fs_status_color='Green'
fi
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=40; spc=''
spc1=${cw}-${#pname}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${pname}$spc: "
echo -e "${idsCL[${fs_status_color}]}${freespace} MB${idsCL[Default]}"
fi
fi
done
nid=`expr $nid + 1`
echo
done
done
fi
fi
########################
########################
if [ "${ST_ACTION}" != "check" ]; then
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
fi
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
}
CHECK_DOCKER_SERVICES(){
for NTYPE in "${NODE_TYPES[@]}"; do
dockers=${NTYPE}_DOCKER[@]
hosts=${NTYPE}_HOSTS[@]
if [[ -v ${NTYPE}_DOCKER ]]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
fi
nid=1
for nip in "${!hosts}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
if [ "${ST_ACTION}" != "check" ]; then
uptime=`${NCMD} uptime -p`
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${FOLDER}/${nip}.down ]; then
if [ -f ${FOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
rm -f ${FOLDER}/${nip}.down
rm -f ${FOLDER}/${nip}.errtime
SENDNOTICE "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
for docker in "${!dockers}"; do
if [ "${ST_ACTION}" != "check" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -aq -f status=exited -f name=${docker})" ]; then
${NCMD} docker rm ${docker}
fi
${NCMD} docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
fi
fi
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
fi
fi
done
else
if [ ! -f ${FOLDER}/${nip}.down ]; then
touch ${FOLDER}/${nip}.down
if [ ! -f ${FOLDER}/${nip}.errtime ]; then
touch ${FOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
if [ "${ST_ACTION}" != "check" ]; then echo; fi
nid=`expr $nid + 1`
done
fi
done
}
NEWCERT(){
echo
if [ -z ${1+x} ]; then
echo -e -n "${idsCL[LightCyan]}Create certificate for what address: ${idsCL[Default]}"
read NEW_CERT
echo
else
NEW_CERT=${1}
fi
if [[ ${NEW_CERT} == *","* ]]; then
IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS
MAIN_CERT=${NEW_CERTS[0]}
else
MAIN_CERT=${NEW_CERT}
fi
if [ "${NEW_CERT}" != "" ]; then
echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}"
echo ""
# echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}"
# for nip in "${WEB_HOSTS[@]}"; do
# if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
# # `ssh root@${nip} service nginx stop`
# # SERVICE nginx stop ${nip}
# fi
# done
# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
# echo
$CERT_DAEMON certonly --webroot -w /opt/lb-data/letsencrypt-acme-challenge -d ${NEW_CERT}
# $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT}
if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then
chown -R root:le /opt/lb-data/letsencrypt
chmod -R 6775 /opt/lb-data/letsencrypt
if [ -f /etc/nginx/sites-enabled/${MAIN_CERT} ]; then
rm -f ${FOLDER}/cert-request.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun
yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun
DIVIDER true
SERVICE nginx reload
fi
echo
echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}"
else
echo
echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}"
fi
# echo
# echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}"
# for nip in "${WEB_HOSTS[@]}"; do
# if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
# `ssh root@${nip} service nginx start`
# # SERVICE nginx start ${nip}
# fi
# done
# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
else
echo "You havent entered a site address."
sleep 1
NEWCERT
exit 0
fi
}
DEL-SSL(){
if [ -z ${1+x} ]; then
echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}"
read DEL_SSL
echo ""
else
DEL_SSL=${1}
fi
if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then
if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then
echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}"
echo ""
echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}"
rm -rf /etc/letsencrypt/archive/${DEL_SSL}
rm -rf /etc/letsencrypt/live/${DEL_SSL}
rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
echo ""
else
echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}"
exit 1
fi
if [ "${timeout}" != "true" ]; then
echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
fi
echo
# SERVICE nginx reload
echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}"
fi
}
CERTRENEW(){
echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}"
ssh root@webserver-node2.scity.us service nginx stop
echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
echo
echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}"
echo
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON renew --force-renewal --preferred-chain "ISRG Root X1" --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
# $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
DIVIDER true
echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}"
ssh root@webserver-node2.scity.us service nginx start
echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}"
echo
SERVICE nginx reload 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun
echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}"
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
}
NIGHTLYRENEW(){
rm -f ${FOLDER}/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx stop
sleep 5
$CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun
CONCAT_SSL
chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun
yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun
ssh root@webserver-node2.scity.us service nginx start
SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun
}
CONCAT_SSL(){
rm -f /tmp/ssllist
for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
for certdir in $(</tmp/ssllist); do
rm -f ${certdir}fullcert.pem
cat ${certdir}privkey.pem ${certdir}fullchain.pem > ${certdir}fullcert.pem
done
}
LISTCERTS(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
CHECKCERT_DOMAINS[${HOST}]=${PORT}
done
unset IFS
if [ ! -z ${LOCAL_SERVICES+x} ]; then
NCMD="ssh root@${MYSQL_HOSTS[0]}"
#${NCMD} rm -f /tmp/ssllist
#${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
else
NCMD=''
rm -f /tmp/ssllist
#for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
fi
echo
echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
DIVIDER false yellow 120
echo -e "Subject Name Monitored Expires Alternate Subject Names"
DIVIDER false yellow 120
cw=30;
c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
ssldir=$(${NCMD} find /opt/nginx-proxy/ssl/* -type l)
for certdir in ${ssldir[@]}; do
SUBJECT=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}/cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}/cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');
SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
c=0; spc=''
spc1=${cw}-${#SUBJECT}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
if [ $DUEIN -gt $CERTEXPIRE ]; then
date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
else
date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
fi
if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
monitored='No '
else
monitored="${idsCL[Green]}Yes${idsCL[Default]}"
fi
if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECTNAMES}"
elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
echo -e "${spc2} ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"
fi
DIVIDER false darkGray 120
done
echo
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
echo ""
}
LISTCERTS_NPM(){
declare -i cw; declare -i spc1; declare -i c
declare -A CHECKCERT_DOMAINS
IFS=$'\n'
for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do
HOST=${LINE%% *}
PORT=${LINE#* }
IFS=" "
CHECKCERT_DOMAINS[${HOST}]=${PORT}
done
unset IFS
if [ ! -z ${LOCAL_SERVICES+x} ]; then
NCMD="ssh root@${MYSQL_HOSTS[0]}"
${NCMD} rm -f /tmp/ssllist
${NCMD} 'for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist
else
NCMD=''
rm -f /tmp/ssllist
for certdir in /opt/nginx-proxy/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist
fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
fi
echo
echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}"
DIVIDER false yellow 120
echo -e "Subject Name Monitored Expires Alternate Subject Names"
DIVIDER false yellow 120
cw=30;
c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
for certdir in $(</tmp/ssllist); do
SUBJECT=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
SUBJECTNAMES=$(${NCMD} openssl x509 -in ${certdir}cert.pem -noout -text|grep -oP '(?<=DNS:|IP Address:)[^,]+'|sort -uV)
CERTEXPIRE=$(date -d "$(: | ${NCMD} openssl x509 -in ${certdir}cert.pem -text | grep 'Not After' |awk '{print $4,$5,$7}')" '+%s');
SUBJECTNAMES=${SUBJECTNAMES//$'\n'/, }
# SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/\n/, /g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}, //g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/, ${SUBJECT}//g")
SUBJECTNAMES=$(echo $SUBJECTNAMES | sed "s/${SUBJECT}//g")
IFS=', '; SUBJECT_NAMES=(${SUBJECTNAMES}); unset IFS
DAYS=14; DUEIN=$(($(date +%s) + (86400*$DAYS)));
c=0; spc=''
spc1=${cw}-${#SUBJECT}
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
if [ $DUEIN -gt $CERTEXPIRE ]; then
date="${idsST[Bold]}${idsCL[Red]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsST[Reset]}${idsCL[Default]}"
else
date="${idsCL[Green]}$(date -d @${CERTEXPIRE} '+%m-%d-%Y')${idsCL[Default]}"
fi
if [ "${CHECKCERT_DOMAINS[${SUBJECT}]}" = "" ]; then
monitored='No '
else
monitored="${idsCL[Green]}Yes${idsCL[Default]}"
fi
if [ ${#SUBJECT_NAMES[@]} -lt 4 ]; then
echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECTNAMES}"
elif [ ${#SUBJECT_NAMES[@]} -lt 7 ]; then
echo -e "${idsCL[Cyan]}${SUBJECT}${idsCL[Default]}${spc}${monitored} ${date} ${SUBJECT_NAMES[0]}, ${SUBJECT_NAMES[1]}, ${SUBJECT_NAMES[2]}"
echo -e "${spc2} ${SUBJECT_NAMES[3]}, ${SUBJECT_NAMES[4]}, ${SUBJECT_NAMES[5]}"
fi
DIVIDER false darkGray 120
done
echo
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
echo ""
}
COPYCERTS_NPM(){
ssldir='/opt/nginx-proxy/ssl'
if [ "${1}" = "reset" ]; then
echo "Resetting ... Deleting all SSL symlink folders"
ssldirs=$(find ${ssldir} -type l)
for certdir in ${ssldirs[@]}; do
rm ${certdir}
done
else
baddirs=$(find $ssldir -type l ! -exec test -e {} \; -print)
for od in ${baddirs[@]}; do
echo "Removing bad symlink folder ... ${od}"
rm -f ${od}
done
fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
fi
ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
for certdir in ${ssls[@]}; do
SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
if [ ! -L ${ssldir}/${SUBJECT} ]; then
echo "${SUBJECT} ... creating"
/bin/ln -s ${certdir} ${ssldir}/${SUBJECT}
fi
done
if [ -z $action ] || [ "${action}" = "gui" ]; then
echo
DIVIDER true
ENTER2CONTINUE
echo
fi
}
CHECK_NPMCERTS(){
ssls=$(find /opt/nginx-proxy/letsencrypt/live/* -type d)
for certdir in ${ssls[@]}; do
SUBJECT=$(openssl x509 -in ${certdir}/cert.pem -noout -subject|grep -oP '(?<=CN = )[^,]+'|sort -uV)
ENDDATE=$(openssl x509 -in ${certdir}/cert.pem -noout -enddate | awk -F '=' '{print $2}')
DAYS=$(dateutils.ddiff today "$(date --date="${ENDDATE}" '+%F')" -f '%d')
c=0; cw=35; spc=''
spc1=`expr ${cw} - ${#SUBJECT}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
[ ${DAYS} -lt 0 ] && spc=${spc::-1}
if [ ${DAYS} -lt 24 ]; then
echo "${SUBJECT}:${spc}${DAYS} days remaining"
fi
done
echo
}
CHECK-CERTS(){
if [ "${1}" = "check" ]; then
rm -f ${FOLDER}/ssl-cert-check.lastrun
${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains > ${FOLDER}/ssl-cert-check.lastrun
if [ "${EMAIL_NOTICE}" != "" ]; then
SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)"
SENDNOTICE "SSL Certs Check" "SSL Certs Check\n$SSLLOG"
fi
else
${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains
fi
}
BACKUP(){
BACKUP_FOLDER=/opt/idssys/backups/node-backup
#BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d`
#BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M`
if [ "${action}" != "backup" ]; then
echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}"
DIVIDER true
fi
if [ ! -d ${BACKUP_FOLDER} ]; then
mkdir -p ${BACKUP_FOLDER}
fi
for item in "${!BACKUP_ITEMS[@]}"
do
if [ "${action}" != "backup" ]; then
echo -en "Backing up '${item}'... "
fi
#tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} .
tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -g ${BACKUP_FOLDER}/backup.snar -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git'
if [ "${action}" != "backup" ]; then
echo -e "${idsCL[Green]}Ok${idsCL[Default]}"
fi
done
touch ${FOLDER}/backup.lastrun
if [ "${action}" != "backup" ]; then
echo ""
DIVIDER true
fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
ENTER2CONTINUE
fi
}
DELSITE(){
while [ $# -gt 0 ]; do
case "$1" in
-site) DEL_SITE=${2};;
-ssl) DEL_SSL=${2};;
-list) DELSITES; exit 0;;
-*)
echo "Invalid option: '${1}' requires an argument" 1>&2
echo ""
echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
width=33
printf "%-${width}s- %s\n" " -site {FQDN address}" "(*required)"
printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well"
printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)"
echo "}"
exit 1;;
esac
shift
done
if [ -z ${DEL_SITE+x} ]; then
echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}"
read DEL_SITE
echo ""
fi
if [[ $DEL_SSL =~ ^[Nn]$ ]]; then
DEL_SSL=no
elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then
DEL_SSL=yes
elif [ -z ${DEL_SSL+x} ]; then
echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} "
read DEL_SSL
fi
if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then
echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}"
echo ""
echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}"
echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}"
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
if [ "${NCMD}" != "" ]; then
checkhost=$(CHECK_HOST ${nip})
fi
if [ "${checkhost}" != "false" ]; then
#if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then
${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE}
${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE}
#fi
#if [ -d /var/www/${DEL_SITE} ]; then
${NCMD} rm -rf /var/www/${DEL_SITE}
#fi
if [ "${DEL_SSL}" = "yes" ]; then
${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE}
${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE}
${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf
fi
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
fi
nid=`expr $nid + 1`
done
echo ""
SERVICE nginx reload ns
echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}"
else
echo "Missing arguments"
echo ""
echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {"
width=33
printf "%-${width}s- %s\n" " -site {FQDN address}" "Site to delete"
printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well"
printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)"
echo "}"
exit 1
fi
}
DELSITES(){
echo
echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}"
DIVIDER true
sid=1
filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )"
# IFS='\n'
for siteconf in $filels; do
# for siteconf in /etc/nginx/sites-available/* ; do
# [ -e "$siteconf" ] || continue
if [ ${siteconf:0:1} == '/' ]; then
IFS='/'; site_conf=(${siteconf}); unset IFS
SITES[${sid}]=${site_conf[4]}
sid=`expr $sid + 1`
fi
done
for s in "${!SITES[@]}"; do
echo " [${s}] ${SITES[${s}]}"
done
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
echo " [B] Back"
fi
echo " [Q] Quit"
echo ""
echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} "
read selsite
echo ""
if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then
echo "Thats an invaild option,"
echo "please select a valid option only."
sleep 1
DELSITES
exit 0
elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then
exit 0
elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then
GUI
else
while :
do
echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} "
read response
echo ""
if [[ $response =~ ^[Yy]$ ]]; then
echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} "
read sslresponse
DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse}
echo ""
DIVIDER
ENTER2CONTINUE
break
else
break
fi
done
DELSITES
exit 0
fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
ENTER2CONTINUE
fi
}
NEWSITE(){
echo
while [ $# -gt 0 ]; do
case "$1" in
-site) NEW_SITE=${2};;
-type) SITE_TYPE=${2};;
-ssl) CREATE_SSL=${2};;
-proxy_scheme) PROXYSCHEME=${2};;
-proxy_host) PROXYHOST=${2};;
-proxy_port) PROXYPORT=${2};;
-h | -help | --help)
echo ""
echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {"
width=33
printf "%-${width}s- %s\n" " -site {FQDN address(,es)}" "(new site and aliases, comma separated)"
printf "%-${width}s- %s\n" " -ssl {yes or no}" "(defaults to yes)"
printf "%-${width}s- %s\n" " -type {'local' or 'proxy'}" "(defaults to local)"
printf "%-${width}s- %s\n" " -proxy_port {host port}" "(proxy backend host)"
printf "%-${width}s- %s\n" " -proxy_host {IP or FQDN}" "(proxy backend port)"
printf "%-${width}s- %s\n" " -sproxy_cheme {http or https}" "(proxy backend scheme)"
echo "}"
exit 1;;
esac
shift
done
#if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi
#if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi
if [ -z ${NEW_SITE+x} ]; then
echo -e -n "${idsCL[LightCyan]}New site domain name: ${idsCL[Default]}"
read NEW_SITE
showdivide=yes
fi
if [ -z ${CREATE_SSL+x} ]; then
echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}"
read CREATE_SSL
showdivide=yes
if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then
CREATE_SSL=yes
else
CREATE_SSL=no
fi
fi
if [ -z ${SITE_TYPE+x} ]; then
echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}"
read SITE_TYPE
showdivide=yes
if [ "${SITE_TYPE}" = "" ]; then
SITE_TYPE=proxy
fi
fi
if [ "${SITE_TYPE}" = "proxy" ]; then
if [ -z ${PROXYHOST+x} ]; then
echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}"
read PROXYHOST
showdivide=yes
fi
if [ -z ${PROXYPORT+x} ]; then
echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}"
read PROXYPORT
showdivide=yes
fi
if [ -z ${PROXYSCHEME+x} ]; then
echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}"
read PROXYSCHEME
showdivide=yes
fi
if [ -z ${WEBOSCKET+x} ]; then
echo -e -n "${idsCL[LightCyan]}Enable Websocket Support (y/N): ${idsCL[Default]}"
read WEBOSCKET
showdivide=yes
if [[ ${WEBOSCKET} =~ ^[Nn]$ ]] || [ "${WEBOSCKET}" = "" ]; then
WEBOSCKET=no
elif [[ ${WEBOSCKET} =~ ^[Yy]$ ]]; then
WEBOSCKET=yes
else
WEBOSCKET=no
fi
fi
if [ -z ${HSTS+x} ]; then
echo -e -n "${idsCL[LightCyan]}Enable HSTS Support (Y/n): ${idsCL[Default]}"
read HSTS
showdivide=yes
if [[ ${HSTS} =~ ^[Yy]$ ]] || [ "${HSTS}" = "" ]; then
HSTS=yes
else
HSTS=no
fi
fi
if [ -z ${EXPLOITS+x} ]; then
echo -e -n "${idsCL[LightCyan]}Block exploits (y/N): ${idsCL[Default]}"
read EXPLOITS
showdivide=yes
if [[ ${EXPLOITS} =~ ^[Nn]$ ]] || [ "${EXPLOITS}" = "" ]; then
EXPLOITS=no
elif [[ ${EXPLOITS} =~ ^[Yy]$ ]]; then
EXPLOITS=yes
else
EXPLOITS=no
fi
fi
fi
[ "${showdivide}" == "yes" ] && DIVIDER
echo
width=14
printf "%-${width}s: %s\n" "New site" "${NEW_SITE}"
printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}"
printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}"
if [ "${SITE_TYPE}" = "proxy" ]; then
printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}"
printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}"
printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}"
printf "%-${width}s: %s\n" "Websocket Support" "${WEBOSCKET}"
printf "%-${width}s: %s\n" "HSTS Support" "${HSTS}"
printf "%-${width}s: %s\n" "Block Exploits" "${EXPLOITS}"
fi
echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} "
read response
echo
if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then
if [[ ${NEW_SITE} == *","* ]]; then
IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS
MAIN_SITE=${NEW_SITES[0]}
NGINX_SERVERNAME=${NEW_SITE//[,]/ }
else
MAIN_SITE=${NEW_SITE}
NGINX_SERVERNAME=${NEW_SITE}
fi
if [ "${SITE_TYPE}" = "proxy" ]; then
if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi
else GO=true
fi
if [ "${GO}" = "true" ]; then
echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
echo ""
######################################### LOCAL
if [ "${SITE_TYPE}" = "local" ]; then
echo -e "server {
listen 80;" > /etc/nginx/sites-available/${MAIN_SITE}
if [ "${CREATE_SSL}" = "yes" ]; then
echo -e " listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE}
fi
echo -e "
server_name ${NGINX_SERVERNAME};
set \$base /var/www/${MAIN_SITE};
root \$base/public_html;
access_log /var/log/nginx/${MAIN_SITE}-access.log;
error_log /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE}
if [ "${CREATE_SSL}" = "yes" ]; then
echo -e "
ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;
include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
fi
echo -e "
index index.php;
location / {
try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE}
if [ "${CREATE_SSL}" = "yes" ]; then
echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
fi
echo -e " }
location ~ \.php\$ {
fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
include conf.d/include/php_fastcgi.conf;
}
include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
if [ "${CREATE_SSL}" = "yes" ]; then
echo -e " include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE}
fi
echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE}
sudo -u www-data mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs}
# echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}"
# for nip in "${WEB_HOSTS[@]}"; do
# checkhost=$(CHECK_HOST ${nip})
# if [ "${checkhost}" != "false" ]; then
# if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then
# checked=false
# until [ "${checked}" = "" ]; do
# checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"`
# done
# fi
# fi
# done
# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
#
# echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}"
# SET-PERMISSIONS ${MAIN_SITE}
# echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
######################################### PROXY
else
[ "${WEBSOCKET}" == "yes" ] && WEBSOCKET="include conf.d/include/websocket-support.conf;" || WEBSOCKET=""
[ "${HSTS}" == "yes" ] && HSTS="include conf.d\/include\/hsts-support.conf;" || HSTS=""
[ "${EXPLOITS}" == "yes" ] && EXPLOITS="include conf.d/include/block-exploits.conf;" || EXPLOITS=""
cp ${FOLDER}/templates/nginx.proxy.site /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<HOSTNAME>>/${MAIN_SITE}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<PROXY_IP>>/${PROXYHOST}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<PROXY_PORT>>/${PROXYPORT}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<PROXY_SCHEME>>/${PROXYSCHEME}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<WEBSOCKET>>/${WEBSOCKET}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<HSTS>>/${HSTS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
sed -i "s/<<EXPLOITS>>/${EXPLOITS}/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
fi
if [ "${CREATE_SSL}" = "yes" ]; then
NEWCERT ${NEW_SITE}
if [ "${SITE_TYPE}" == "proxy" ]; then
sed -i "s/#ssl_certificate/ssl_certificate/g" /etc/nginx/sites-enabled/${MAIN_SITE}.conf
fi
fi
rm -f ${FOLDER}/new-site.lastrun
daterun=`date +%Y-%m-%d-%H-%M-%S`
echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun
# yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun
# daterun=`date +%Y-%m-%d-%H-%M-%S`
# echo -e "${daterun}" >> /etc/nginx/new-site.lastrun
DIVIDER true
echo ""
echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}"
echo ""
else
echo "Missing proxy arguments"
Exit 1
fi
else
${SCRIPT} newsite
exit 0
fi
}
SET-PERMISSIONS(){
if [ "${1}" = "gitea" ]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
${NCMD} chown -R git:git /var/lib/gitea
${NCMD} chmod -R 750 /var/lib/gitea
${NCMD} chown -R git:git /home/git
${NCMD} chmod -R 750 /home/git
if [ "${2}" != "q" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
fi
elif [ "${2}" != "q" ]; then
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
fi
nid=`expr $nid + 1`
done
elif [ "${1}" = "pdnsadmin" ]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on Webserver-Node${nid} (${nip})... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
${NCMD} chown -R www-data:www-data /opt/powerdns-admin
${NCMD} chmod -R 775 /opt/powerdns-admin
if [ "${2}" != "q" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
fi
elif [ "${2}" != "q" ]; then
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
fi
nid=`expr $nid + 1`
done
elif [ "${1}" = "nextcloud" ]; then
nip=${NC_HOSTS[0]}
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on Nextcloud-Node1 (${nip})... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
${NCMD} chown -R www-data:www-data /mnt/Nextcloud-Data
${NCMD} find /mnt/Nextcloud-Data/ -type d -exec chmod 770 {} \;
${NCMD} find /mnt/Nextcloud-Data/ -type f -exec chmod 660 {} \;
if [ "${2}" != "q" ]; then
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
fi
elif [ "${2}" != "q" ]; then
echo -e "${idsCL[Red]}Node is down${idsCL[Default]}"
fi
else
DIVIDER
if [ "${1}" != "" ]; then
NEW_SITE=${1}
echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}"
fi
if [ "${NEW_SITE}" = "xxx" ]; then
nid=1
for nip in "${MYSQL_HOSTS[@]}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -e "${idsCL[Yellow]}Setting folder permissions for MySQL-Node${nid} (${nip})${idsCL[Default]}"
echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}"
${NCMD} chown -R root:letsencrypt /etc/letsencrypt
${NCMD} chmod -R 6775 /etc/letsencrypt
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo ""
fi
nid=`expr $nid + 1`
done
fi
nid=1
for nip in "${WEB_HOSTS[@]}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${NEW_SITE}" != "" ]; then
echo -en "${idsCL[Cyan]}Webserver-Node${nid} (${nip}) ${idsCL[Default]}"
${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE
${NCMD} chmod -R 7775 /var/www/$NEW_SITE
${NCMD} chown -R www-data:www-data /var/lib/php/sessions
${NCMD} chmod -R 7777 /var/lib/php/sessions
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
else
echo -e "${idsCL[Yellow]}Setting folder permissions for Webserver-Node${nid} (${nip})${idsCL[Default]}"
echo -en "${idsCL[Cyan]}Webserver folders ${idsCL[Default]}"
${NCMD} chown -R www-data:www-data /var/www
${NCMD} chmod -R 7775 /var/www
${NCMD} chown -R www-data:www-data /var/lib/php/sessions
${NCMD} chmod -R 7777 /var/lib/php/sessions
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo -en "${idsCL[Cyan]}Gitea Folder ${idsCL[Default]}"
${NCMD} chown -R git:git /var/lib/gitea
${NCMD} chmod -R 750 /var/lib/gitea
${NCMD} chown -R git:git /home/git
${NCMD} chmod -R 750 /home/git
echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}"
echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}"
${NCMD} chown -R root:letsencrypt /etc/letsencrypt
${NCMD} chmod -R 6775 /etc/letsencrypt
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo -en "${idsCL[Cyan]}PowerDNS-Admin ${idsCL[Default]}"
${NCMD} chown -R www-data:www-data /opt/powerdns-admin
${NCMD} chmod -R 775 /opt/powerdns-admin
echo -e "${idsCL[Green]}Complete${idsCL[Default]}"
echo ""
fi
fi
nid=`expr $nid + 1`
done
if [ "${NEW_SITE}" != "" ]; then echo ""; fi
if [ -z $action ] || [ "${action}" = "gui" ]; then
ENTER2CONTINUE
fi
fi
}
SERVICES(){
while :
do
echo ""
echo -e "${idsCL[Green]}Node Services${idsCL[Default]}"
DIVIDER true
s=1
for srvc in "${NODE_SERVICES[@]}"
do
echo " [${s}] ${NM_SERVICES[${srvc}]}"
s=`expr $s + 1`
done
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
echo " [B] Back"
fi
echo " [Q] Quit"
echo ""
echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} "
read -n 1 selsrvc
selsrvc=`expr $selsrvc - 1`
echo ""
if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then
echo "Thats an invaild option,"
echo "please select a valid option only."
sleep 1
SERVICES
exit 0
elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
exit 0
elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
GUI
else
echo ""
echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}"
DIVIDER true
s=1
for srvcopt in "${NM_SRVCOPT[@]}"; do
echo " [${s}] ${srvcopt~}"
s=`expr $s + 1`
done
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
echo " [B] Back"
fi
echo " [Q] Quit"
echo ""
echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} "
read -n 1 selsrvcopt
selsrvcopt=`expr $selsrvcopt - 1`
echo ""
if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${selsrvcopt}" != "B" ] && [ "${selsrvcopt}" != "b" ]; then
echo "Thats an invaild option,"
echo "please select a valid option only."
sleep 1
SERVICES
exit 0
elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then
exit 0
elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then
SERVICES
exit 0
else
echo
SERVICE ${NODE_SERVICES[${selsrvc}]} ${NM_SRVCOPT[${selsrvcopt}]}
echo ""
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER
ENTER2CONTINUE
fi
break
fi
fi
done
}
SERVICE(){
NODE_TYPE=''
#var=NC_HOSTS[@]
if [ "$1" = "glusterd" ]; then
for nip in "${GL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD systemctl $2 $1 > /dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
if [ "${2}" = "start" ] || [ "${2}" = "restart" ]; then
echo
sleep 5s
for nip in "${GL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
if [ "${3}" != "q" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
#$NCMD mount -a > /dev/null 2>&1
if [ "${3}" != "q" ]; then
if [[ "${NCHOSTS}" == *"${nip}"* ]]; then
mntpath="/mnt/Nextcloud-Data"
elif [[ "${WEBHOSTS}" == *"${nip}"* ]]; then
mntpath="/mnt/web-data"
fi
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
if [ "$nip" = "10.10.1.43" ]; then
echo -en "Mounting drive on '$nip'... ${idsCL[Default]}"
mntpath="/mnt/Nextcloud-Data"
${NCMD} "test -e ${mntpath}/mounted"
if [ $? -eq 0 ]; then
echo -e "${idsCL[Green]}MOUNTED ($mntpath)${idsCL[Default]}"
else
echo -e "${idsCL[Red]}NOT MOUNTED ($mntpath)${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
done
fi
else
IP=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1)
for NTYPE in "${NODE_TYPES[@]}"; do
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
NODE_TYPE=${NTYPE};
break 2
fi
done
done
if [ "$NODE_TYPE" = "NC" ]; then
NT_HOSTS=NC_HOSTS[@]
NT_HOST=NC_HOSTS[0]
NTS=('NC'); #NTS=$NTS1[@]
elif [ "$NODE_TYPE" = "LB" ]; then
NT_HOSTS=LB_HOSTS[@]
NT_HOST=LB_HOSTS[0]
NTS=('LB'); #NTS=$NTS1[@]
elif [ "$NODE_TYPE" = "WEB" ]; then
NT_HOSTS=WEB_HOSTS[@]
NT_HOST=WEB_HOSTS[0]
NTS=('WEB'); #NTS=$NTS1[@]
elif [ "$NODE_TYPE" = "MYSQL" ]; then
NT_HOSTS=MYSQL_HOSTS[@]
NT_HOST=MYSQL_HOSTS[0]
NTS=('MYSQL'); #NTS=$NTS1[@]
elif [ "${1}" = "nginx" ]; then
NT_HOSTS=WEB_HOSTS[@]
NT_HOST=WEB_HOSTS[0]
NTS=('WEB'); #NTS=NTS1[@]
# NTS=$NODE_TYPES
else
NTS=(${NODE_TYPES[*]})
fi
if [ "${3}" != "q" ]; then
if [ "${NM_SERVICES[${1}]}" = "" ]; then
echo -e "${idsCL[Red]}(${1}) is not an allowed service.${idsCL[Default]}"
exit 1
fi
if [ "${NM_SRVCOPTS[${2}]}" = "" ]; then
echo -e "${idsCL[Red]}(${2}) is not an allowed service action.${idsCL[Default]}"
exit 1
fi
if [ "$2" = "start" ] && [ "$1" = "haproxy" ]; then
TACT='restart'
else
TACT="${2}"
fi
echo
echo -e "${idsCL[LightGreen]}[[ ${NM_SERVICES[${1}]} ${NM_SRVCOPTS[${2}]}ing ]]${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
fi
if [ "${1}" = "nginx-x" ]; then
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying nginx config and ssl cert replication across the nodes... ${idsCL[Default]}"
fi
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${!NT_HOST}"* ]]; then
PH_CMD="ssh root@localhost"
else
PH_CMD="ssh root@${!NT_HOST}"
fi
${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/nginx/test.repl"
# if [ "$NODE_TYPE" != "NC" ]; then
# ${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/letsencrypt/test.repl"
# fi
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "" ]; do
checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/nginx/test.repl ]; then cat /etc/nginx/test.repl; fi' | diff - /etc/nginx/test.repl\"`"
# checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/nginx/test.repl ]; then cat /etc/letsencrypt/test.repl; fi' | diff - /etc/letsencrypt/test.repl\"`"
done
fi
fi
done
${PH_CMD} rm -f /etc/nginx/test.repl
# if [ "$NODE_TYPE" != "NC" ]; then
# ${PH_CMD} rm -f /etc/letsencrypt/test.repl
# fi
fi
fi
elif [ "${1}" = "apache2-x" ]; then
if [ "${2}" = "restart" ] || [ "${2}" = "reload" ]; then
if [ "${3}" != "ns" ]; then
if [ "${3}" != "q" ]; then
echo -en "${idsCL[LightYellow]}Verifying Apache config and ssl cert replication across the nodes... ${idsCL[Default]}"
fi
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${!NT_HOST}"* ]]; then
PH_CMD="ssh root@localhost"
else
PH_CMD="ssh root@${!NT_HOST}"
fi
echo "HERE: '$PH_CMD'"
${PH_CMD} "echo -e \"Service ${1} ${2}\" > /etc/apache2/test.repl"
for nip in "${!NT_HOSTS}"; do
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
if [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "" ]; do
checked="`${PH_CMD} \"ssh root@${nip} 'if [ -f /etc/apache2/test.repl ]; then cat /etc/apache2/test.repl; fi' | diff - /etc/apache2/test.repl\"`"
done
fi
fi
done
${PH_CMD} rm -f /etc/apache2/test.repl
fi
fi
fi
if [ "${3}" != "q" ]; then
echo -e "${idsCL[Green]}Completed${idsCL[Default]}"
echo
fi
for NTYPE in "${NTS[@]}"; do
var1=${NTYPE}SERVICES_CHECK
if [[ "${!var1}" = *"${1}"* ]]; then
nid=1
if [ "${3}" != "" ]; then
var2=(${3})
sethost=true;
else
var2=${NTYPE}_HOSTS[@]
IFS=' '
var2=(${!var2})
unset IFS
sethost=false
fi
for nip in "${var2[@]}"; do
# if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${3}"* ]]; then GO=true;
# elif [ ! -z ${3+x} ] || [ "${3}" == "q" ]; then GO=true;
# else GO=false;
# fi
# if [ "${GO}" == "true" ]; then
if [ $sethost == true ]; then
nodename=${NM_HOSTNAMES[${3}]}
else
nodename="${NM_NODETYPES[$NTYPE]}-Node${nid}"
fi
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
NOGOCHK=true;
# if [ "${1}" == "gitea" ] || [ "${1}" == "pdnsadmin" ] || [ "${1}" == "pdnsadmin.socket" ]; then
if [ "${1}" == "gitea" ]; then
if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.10.1.120* ]]; then
NOGOCHK=false;
fi
fi
if [ "${NOGOCHK}" == "true" ]; then
if [ "${3}" != "q" ]; then
echo -en "${NM_SRVCOPTS[${2}]}ing on ${nodename} ($nip)... ${idsCL[Default]}"
fi
checkhost=$(CHECK_HOST ${nip})
if [ "${checkhost}" != "false" ]; then
$NCMD systemctl $2 $1
if [ "${3}" != "q" ]; then
if [[ "enable,disable" = *"${2}"* ]] || [ "${1}" = "daemon-reload" ]; then
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
elif [ "$(${NCMD} systemctl is-active ${1})" = "active" ]; then
echo -e "${idsCL[Green]}OK${idsCL[Default]}"
elif [ "$2" = "stop" ]; then
echo -e "${idsCL[Red]}STOPPED${idsCL[Default]}"
else
echo -e "${idsCL[Red]}ERROR${idsCL[Default]}"
fi
fi
elif [ "${3}" != "q" ]; then
echo -e "${idsCL[Red]}Node is Down${idsCL[Default]}"
fi
fi
# fi
nid=`expr $nid + 1`
# if [ "${1}" = "gitea" ]; then
# break
# fi
done
fi
done
fi
if [ "${3}" != "q" ]; then
echo
echo -e "${idsCL[Green]}${NM_SERVICES[${1}]} has been ${NM_SRVCOPTS[${2}]}ed${idsCL[Default]}"
echo
fi
}
HAPROXY-CONFIG() {
echo -e "${idsCL[Yellow]}Creating HAProxy Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"haproxy"* ]]; then
nid=1
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
echo
fi
if [[ "${WEBSERVICES_CHECK}" = *"haproxy"* ]]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
fi
if [[ "${LBSERVICES_CHECK}" = *"haproxy"* ]]; then
nid=1
for nip in "${LB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Creating File on LB-Node${nid} ($nip)... ${idsCL[Default]}"
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
fi
SERVICE haproxy reload
echo ""
echo -e "${idsCL[Green]}HAProxy Config Files have been created.${idsCL[Default]}"
echo ""
}
SINGLE_SERVER_SERVICE_CHECK(){
if [ "${1}" = "web" ]; then
# servicestocheck=gitea,pdnsadmin
servicestocheck=gitea
IFS=,; services_to_check=(${servicestocheck}); unset IFS
local_ips=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1);
local_ip=$(/sbin/ip -o -4 addr list eth0 |grep 255 | awk '{print $4}' | cut -d/ -f1)
if [[ ${local_ips} = *10.10.1.120* ]]; then
if [ -f /mnt/web-data/mounted ]; then
for stc in "${services_to_check[@]}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
for nip in "${WEB_HOSTS[@]}"; do
if [[ "${local_ip}" != *"${nip}"* ]]; then
ssh root@${nip} /bin/systemctl stop ${stc}
fi
done
sleep 3s
/bin/systemctl start ${stc} &
fi
done
for nip in "${WEB_HOSTS[@]}"; do
if [[ "${local_ip}" != *"${nip}"* ]]; then
ssh root@${nip} /usr/bin/docker stop vaultwarden >/dev/null 2>&1
fi
done
/usr/bin/docker start vaultwarden & >/dev/null 2>&1
fi
else
for stc in "${services_to_check[@]}"; do
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
/bin/systemctl stop ${stc} &
done
/usr/bin/docker stop vaultwarden & >/dev/null 2>&1
fi
if [ "$(systemctl is-active nginx)" != "active" ]; then
/usr/local/bin/nodemgmt service web restart ${local_ip}
fi
fi
}
KEEPALIVE-CONFIG() {
echo -e "${idsCL[Yellow]}Creating Keepalive Config Files${idsCL[Default]}"
DIVIDER false yellow
if [[ "${MYSQLSERVICES_CHECK}" = *"keepalive"* ]]; then
nid=1
for nip in "${MYSQL_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Creating File on MySQL-Node${nid} ($nip)... ${idsCL[Default]}"
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
echo
fi
if [[ "${WEBSERVICES_CHECK}" = *"keepalive"* ]]; then
nid=1
for nip in "${WEB_HOSTS[@]}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
NCMD=''
else
NCMD="ssh root@${nip}"
fi
echo -en "Creating File on Webserver-Node${nid} ($nip)... ${idsCL[Default]}"
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
fi
SERVICE keepalived reload
echo ""
echo -e "${idsCL[Green]}Keepalive Config Files have been created.${idsCL[Default]}"
echo ""
}
NODEUPDATE() {
start=`date +%s`
if [ "${1}" = "report" ]; then
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
nip=${MYSQLMANAGER}
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost'
fi
c=0; cw=14; spca=''
spc1=${cw}-7
until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
c=0; cw=15; spcb=''
spc1=${cw}-${#nip}
until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
echo -en "Updating MySQL-Manager${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
ssh root@${nip} "nodemgmt update q"
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
ssh root@${nip} "runup runup"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
echo
for NTYPE in "${NODE_TYPES[@]}"; do
nid=1
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then
nip='localhost '
fi
NCMD="ssh root@${nip}"
c=0; cw=14; spca=''
spc1=${cw}-${#NM_NODETYPES[$NTYPE]}
until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
c=0; cw=15; spcb=''
spc1=${cw}-${#nip}
until [ $c = ${spc1} ]; do spcb="${spcb} "; c=`expr $c + 1`; done
echo -en "Updating ${NM_NODETYPES[$NTYPE]}-Node${nid}${spca}($nip)${spcb}... ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
${NCMD} "nodemgmt update q"
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ... ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
${NCMD} "runup runup"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
echo
done
echo -e "${idsCL[Green]}All Nodes have been Updated.${idsCL[Default]}"
echo
echo
end=`date +%s`
runtime=$((end-start))
echo "runtime: ${runtime}"
}
VCENTER-SSL(){
VCHOSTNAME='vcenter.scity.us'
#BASECERTDIR="/etc/letsencrypt/live"
BASECERTDIR="/opt/nginx-proxy/ssl"
CERTDIR="${BASECERTDIR}/${VCHOSTNAME}"
SERVER="https://${VCHOSTNAME}"
CRED="administrator@scity.vs:$(pass iDSVC)"
VCUSER="administrator@scity.vs"
CURRENTLIVE=/etc/vmware-rhttpproxy/ssl/rui.crt
CERT=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.cer
KEY=/root/.acme.sh/$VCHOSTNAME/$VCHOSTNAME.key
CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
VCCERTDAYS=${VCCERTDAYS#*=}
if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
echo -e "${idsCL[Green]}${VCCERTDAYS} days left, Certificate is still valid, no noeed to update${idsCL[Default]}"
echo
exit 0
elif [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" == "force" ]; then
echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, forcing certificate update${idsCL[Default]}"
echo
else
echo -e "${idsCL[Yellow]}${VCCERTDAYS} days left, Certificate needs to be updated${idsCL[Default]}"
echo
fi
LIVEMD5=$(ssh root@$VCHOSTNAME "md5sum $CURRENTLIVE | cut -d\ -f1")
CURRENTMD5=$(md5sum $CERTDIR/cert.pem | cut -d\ -f1)
if [ "$LIVEMD5" == "$CURRENTMD5" ] && [ "${1}" != "force" ]; then
echo -e "${idsCL[Yellow]}Certificates remains the same, no newer certificates exist${idsCL[Default]}"
echo
exit 0
fi
echo -e "${idsCL[Green]}Updating certificates on vCenter... ${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}This process make take up to 10mins${idsCL[Default]}"
echo
# eval $(awk '{ print "export " $1 }' /etc/sysconfig/vmware-environment)
scp $BASECERTDIR/$VCHOSTNAME/cert.pem root@${VCHOSTNAME}:$CERT
scp $BASECERTDIR/$VCHOSTNAME/privkey.pem root@${VCHOSTNAME}:$KEY
scp $BASECERTDIR/$VCHOSTNAME/fullchain.pem root@${VCHOSTNAME}:$CHAIN
ssh root@$VCHOSTNAME "(printf '1\n%s\n' '$VCUSER'; sleep 1; printf '%s\n' '$(pass iDSVC)'; sleep 1; printf '2\n'; sleep 1; printf '%s\n%s\n%s\ny\n\n' '$CERT' '$KEY' '$CHAIN') | setsid /usr/lib/vmware-vmca/bin/certificate-manager"
# SESSION_ID=$(curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' --header 'vmware-use-header-authn: test' --header 'vmware-api-session-id: null' -u $CRED "$SERVER/rest/com/vmware/cis/session" --insecure | python3 -c "import sys, json; print(json.load(sys.stdin)['value'])")
# PRIVKEY=$(awk -v ORS='\\n' '1' "$CERTDIR/privkey.pem")
# CERT=$(awk -v ORS='\\n' '1' "$CERTDIR/fullcert.pem")
# REQUEST_BODY="{ \"spec\" : { \"cert\" : \"$CERT\", \"key\" : \"$PRIVKEY\" } }"
# curl --insecure -X PUT "$SERVER/rest/vcenter/certificate-management/vcenter/tls" \
# -H "vmware-api-session-id: $SESSION_ID" \
# -H "Content-type: application/json" \
# -d "$REQUEST_BODY"
# curl --insecure -X DELETE "$SERVER/rest/com/vmware/cis/session" -H "vmware-api-session-id: $SESSION_ID"
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
# echo
# read -n 1 -s -p "Press any key to continue"
# echo -en "${idsCL[LightCyan]}Stopping vCenter services... ${idsCL[Default]}"
# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --stop --all' >/dev/null 2>&1
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
#
# echo -en "${idsCL[LightCyan]}Starting vCenter services... ${idsCL[Default]}"
# ssh -i ~/.ssh/id_rsa root@vcenter.scity.us 'service-control --start --all' >/dev/null 2>&1
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo
echo -e "${idsCL[Green]}The vCenter certifcate has been updated${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}Don't forget to re-scan the vCenter connection in Veeam${idsCL[Default]}"
echo
}
ADD_LOGROTATE_CRONTAB(){
if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then
(crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab -
fi
/bin/chmod 0644 ${FOLDER}/tmp-logrotate
echo ""
echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
echo ""
read -n 1 -s -p "Press any key to continue"
echo
}
UPDATE_DYNDNS(){
if [ "${1}" != 'q' ]; then
echo -e "${idsCL[LightCyan]}Updating Primary DNS A records with any WAN IP changes... ${idsCL[Default]}"
fi
OLDIP=`dig +short sc1.scity.us @ns1.scity.us`
NEWIP=`dig +short myip.opendns.com @resolver1.opendns.com`
if [ "${OLDIP}" != "${NEWIP}" ] && [ "${NEWIP}" != ";; connection timed out; no servers could be reached" ]; then
dnss=0
for i in {1..3}; do
curl -m 15 -s --head --request GET https://wdns${i}.scity.us | grep "HTTP/2 302" > /dev/null && dnss=$i && break
done
if [ "${dnss}" != "0" ]; then
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=sc1.scity.us&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=sc2.scity.us&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=scity.us&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=schroedercity.com&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=dasrei.com&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=snsinvest.com&myip=${NEWIP}" > /dev/null 2>&1
/usr/bin/curl -s "https://systems:Dcs9613@wdns${dnss}.scity.us/nic/update?hostname=misfitsmc.com&myip=${NEWIP}" > /dev/null 2>&1
SENDNOTICE "Updated WAN IP from '${OLDIP}' to '${NEWIP}'" "DYNDNS Updated"
if [ "${1}" != 'q' ]; then
echo -e "${idsCL[Green]}Updated WAN IP from '${OLDIP}' to '${NEWIP}'${idsCL[Default]}"
fi
else
SENDNOTICE "Could not connect to any WDNS-PDNS server to update WAN IP" "DYNDNS ERROR"
fi
else
if [ "${1}" != 'q' ]; then
echo -e "${idsCL[Green]}WAP IP has not changed '${OLDIP}' = '${NEWIP}'${idsCL[Default]}"
fi
fi
if [ "${1}" != 'q' ]; then
echo ""
read -n 1 -s -p "Press any key to continue"
echo
fi
}
RUN_COMMAND(){
echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
for NTYPE in "${NODE_TYPES[@]}"; do
echo -e "${idsST[Bold]}"; DIVIDER
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Nodes to run command { \`${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}\` }${idsCL[Default]}"
DIVIDER; echo -e "${idsST[Reset]}"
nid=1
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh root@${nip}"; LH=''
fi
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
#echo "Running command: ${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}"
${NCMD} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14}
DIVIDER false green
echo
nid=`expr $nid + 1`
done
echo
done
}
DOWNLIST(){
if [ "${1}" = "report" ]; then
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
DOWN_LIST=`find ${FOLDER} -name "*.down"`
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
fi
echo
if [ "${DOWN_LIST}" == "" ]; then
echo -e "${idsCL[LightCyan]}Nothing is down${idsCL[Default]}"
else
echo -e "${idsCL[LightGreen]}Current List of Items Down...${idsCL[Default]}"
DIVIDER false yellow
echo -e "Service/Host Host Down_Time"
DIVIDER false yellow
cw=40; cw2=58
for downitem in ${DOWN_LIST}; do
ditem=${downitem##*/}
ditem=${ditem//.down}
host=${ditem%%-*}
item=${ditem#*-}
if [[ $ditem == *"-"* ]] && [ "${NM_SERVICES[${item}]}" != "" ]; then
itemname=${NM_SERVICES[${ditem#*-}]}
elif [[ $ditem == *"-"* ]] && [ "${NM_DOCKERS[${item}]}" != "" ]; then
itemname="${NM_DOCKERS[${ditem#*-}]} (docker)"
elif [[ $ditem == *"-"* ]] && [ "${REPL_DESC[${item}]}" != "" ]; then
itemname="${REPL_DESC[${item}]} (replication)"
else
itemname='HOST DOWN';
fi
if [ -f ${FOLDER}/${ditem}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${ditem}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${ditem}.down)
fi
c=0; spc=''
spc1=${cw}-${#itemname}
#echo "HERE: $downitem ~ $ditem ~ $host ~ $item ~ $itemname ~ $spc1"
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
c=0; spc2=''
spc1=${cw2}-${#itemname}-${#host}-${spc1}
until [ $c = ${spc1} ]; do spc2="${spc2} "; c=`expr $c + 1`; done
echo -e "${idsST[Bold]}${idsCL[Red]}${itemname}${spc}${host}${spc2}$(SHOW_TIME ${toterrtime})${idsST[Reset]}${idsCL[Default]}"
DIVIDER false darkGray
done
fi
echo
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
ENTER2CONTINUE
fi
echo ""
}
UPDATEPUSHOVERINFO(){
echo
if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
echo -en "${idsCL[LightCyan]}Use current Pushover Info:\nApp Token: [${idsCL[White]}${PUSHOVER_APP_TOKEN}${idsCL[LightCyan]}\nUser Token: [${idsCL[White]}${PUSHOVER_USER_TOKEN}${idsCL[LightCyan]}] (Y/n)? ${idsCL[Default]}"
read -n 1 choice
case "$choice" in
[Nn]) PUSHOVER_APP_TOKEN="";PUSHOVER_USER_TOKEN="";;esac
fi
if [ "${PUSHOVER_APP_TOKEN}" == "" ]; then
echo; echo -e -n "${idsCL[LightCyan]}Pushover App Token: ${idsCL[Default]}"
read -e PUSHOVER_APP_TOKEN
echo; echo -e -n "${idsCL[LightCyan]}(optional) Pushover User Token: ${idsCL[Default]}"
read -e PUSHOVER_USER_TOKEN
fi
declare -A host_ip
declare -A host_name
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
host_name[${hostid}]=$hostname
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
for hostid in "${!host_ip[@]}"; do
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running commands${idsCL[Default]}${idsST[Reset]}"
ssh root@${host_ip[$hostid]} "/bin/sed -i '/PUSHOVER_/d' /opt/idssys/nodemgmt/settings.conf"
if [ "${PUSHOVER_APP_TOKEN}" != "" ]; then
echo "putting in info 'root@${host_ip[$hostid]}'"
ssh root@${host_ip[$hostid]} "echo PUSHOVER_APP_TOKEN=\\\"${PUSHOVER_APP_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes1'"
else
echo "taking out info"
ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_APP_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
fi
if [ "${PUSHOVER_USER_TOKEN}" != "" ]; then
ssh root@${host_ip[$hostid]} "echo PUSHOVER_USER_TOKEN=\\\"${PUSHOVER_USER_TOKEN}\\\" >> ${MMFOLDER}/settings.conf; echo 'yes2'"
else
ssh root@${host_ip[$hostid]} "echo \# PUSHOVER_USER_TOKEN=\\\"\\\" >> ${MMFOLDER}/settings.conf"
fi
else
echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
fi
done
}
GUI(){
DISP_HEADER true false
while :
do
echo " [1] Node Status"
echo " [2] Node Services"
echo ""
echo " [3] New Site"
echo " [4] Delete Site"
echo ""
echo " [5] New Certificate"
echo " [6] Renew Certificates"
echo " [7] List Certificates"
echo ""
echo " [8] Set folder permissions on nodes"
echo " [9] Backup Node Settings and Files"
echo ""
echo " [D] Update Primary DNS Records"
echo " [R] Add cron task to rotate wtmp & btmp log files"
echo ""
echo " [Q] Quit"
echo ""
echo ""
echo -e -n "${idsCL[LightYellow]}Please select an [ActionItem] from above:${idsCL[Default]} "
read -n 1 opt
echo ""
case $opt in
1) STATUS; GUI;;
2) SERVICES; GUI;;
3) NEWSITE; GUI;;
4) DELSITES; GUI;;
5) NEWCERT; GUI;;
6) CERTRENEW; GUI;;
7) LISTCERTS; GUI;;
8) SET-PERMISSIONS; GUI;;
9) BACKUP; GUI;;
[Dd]) UPDATE_DYNDNS; GUI;;
[Rr]) ADD_LOGROTATE_CRONTAB; GUI;;
[Qq]) EXIT1
exit 0;;
*) echo "Thats an invaild option,";
echo "please select a valid option only.";
sleep 1;;
esac
done
}
if [ ${action-x} ]; then
if [[ "${noheader}" != *" ${action} "* ]] && [[ "${noheader}" != *" ${2} "* ]] && [ "${2}" != "q" ] && [ "${1}" != "gui" ] && [ "${1}" != "nightlyrenew" ]; then
DISP_HEADER
fi
case $action in
newcert) NEWCERT ${2};;
certrenew) CERTRENEW;;
delssl) DEL-SSL ${2};;
listcerts) LISTCERTS;;
listcerts-npm) LISTCERTS_NPM;;
copynpmcerts) COPYCERTS_NPM ${2};;
checknpmcerts) CHECK_NPMCERTS;;
checkcerts) CHECK-CERTS ${2};;
nightlyrenew) NIGHTLYRENEW;;
delsite) DELSITE ${2} ${3} ${4} ${5};;
delsites) DELSITES;;
newsite) NEWSITE ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15};;
update) ;;
update-nodes) NODEUPDATE ${2};;
update-dyndns) UPDATE_DYNDNS ${2};;
backup) BACKUP;;
downlist)
if [ "${2}" = "report" ]; then
if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then
rm -f ${FOLDER}/status-downlist.lastrun
DOWNLIST report >> ${FOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${FOLDER}/status-downlist.lastrun)"
SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
fi
else
DOWNLIST ${2}
fi;;
status)
if [ -f "${FOLDER}/status-check.running" ]; then
fd=`stat --format=%Y "${FOLDER}/status-check.running"`
cbd=$((`date +%s` - 450))
[ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running"
fi
if [ ! -f "${FOLDER}/status-check.running" ]; then
touch "${FOLDER}/status-check.running"
if [ "${2}" = "report" ]; then
rm -f ${FOLDER}/status-check.lastrun
STATUS report ${3} > ${FOLDER}/status-check.lastrun
if [ "${3}" = "email" ] && [ "${EMAIL_NOTICE}" != "" ]; then
echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
DOWNLIST report >> ${FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
fi
else
STATUS ${2} ${3}
fi
rm -f "${FOLDER}/status-check.running"
else
echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
echo
fi;;
checkdocker) CHECK_DOCKER_SERVICES ${2};;
status-check) STATUS-CHECK ${2};;
set-permissions) SET-PERMISSIONS ${2};;
service)
if [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "start" ]; then
#SET-PERMISSIONS ${2}
SERVICE ${2} start
elif [[ ( "$2" = "giteax" ) || ( "$2" = "powerdns-admin" ) ]] && [ "$3" = "restart" ]; then
SERVICE ${2} stop
#SET-PERMISSIONS ${2}
SERVICE ${2} start
elif [ "$2" = "web" ]; then
SERVICE php5.6-fpm ${3} ${4}
SERVICE php7.2-fpm ${3} ${4}
SERVICE php7.3-fpm ${3} ${4}
SERVICE php7.4-fpm ${3} ${4}
SERVICE php8.2-fpm ${3} ${4}
SERVICE nginx ${3} ${4}
elif [ "$2" = "glusterdxx" ]; then
SERVICE glusterd stop
else
SERVICE ${2} ${3} ${4}
fi;;
services) SERVICES;;
singleservercheck) SINGLE_SERVER_SERVICE_CHECK ${2} ${3};;
haproxy-config) HAPROXY-CONFIG;;
keepalive-config) KEEPALIVE-CONFIG;;
vc-ssl) VCENTER-SSL ${2};;
run) RUN_COMMAND ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15};;
update-pushover) UPDATEPUSHOVERINFO;;
gui) GUI;;
*)
echo -e "Usage: ${idsCL[LightCyan]}nodemgmt${idsCL[Default]} {"
echo -e " ${idsCL[Yellow]}backup${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}certrenew${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}checkcerts${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}delsite${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}delsites${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}delssl${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}gui${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}haproxy-config${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}listcerts${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}newcert${idsCL[Default]} [domain(,s)]"
echo -e " ${idsCL[Yellow]}newsite${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}nightlyrenew${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}run${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}service${idsCL[Default]} [service] [action]"
echo -e " ${idsCL[Yellow]}services${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}set-permissions${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}status${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}status-check${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}update-nodes${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}update-dyndns${idsCL[Default]}"
echo -e " ${idsCL[Yellow]}vc-ssl${idsCL[Default]}"
echo -e "}"
echo ""
echo ""
exit 0;;
esac
else
GUI
fi
exit 0
Reference in New Issue View Git Blame Copy Permalink