#!/usr/bin/env bash # NodeMgmt - Galera/NGINX Node Management Scripts action="$1" if [ ! -f /opt/idssys/settings/nodemgmt.conf ]; then if [ ! -d /opt/idssys/settings ]; then mkdir /opt/idssys/settings fi mv /opt/idssys/nodemgmt/settings.conf /opt/idssys/settings/nodemgmt.conf fi source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/settings/nodemgmt.conf source /opt/idssys/nodemgmt/defaults.inc #source /opt/idssys/nodemgmt/ssl-check.sh # ========================================================= # STATUS(){ start=`date +%s` ST_ACTION=${1} if [ "${ST_ACTION}" = "report" ]; then unset idsCL idsBG idsST idsCL=('') idsBG=('') idsST=('') fi declare -i cw; declare -i spc1; declare -i c ######################## if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -en "${idsCL[LightCyan]}Setting up status checks... ${idsCL[Default]}" fi #NHCMD="ssh root@${MYSQL_HOSTS[0]}" #LBHCMD="ssh root@${LB_HOSTS[0]}" #WHCMD="ssh root@${WEB_HOSTS[0]}" for NTYPE in "${NODE_TYPES[@]}"; do PH=${NTYPE}_HOSTS[0] PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do ${PH_CMD} rm -f ${FOLDER}/test.repl daterun=`date +%Y-%m-%d-%H-%M-%S` if [ "${PH_CMD}" = "" ]; then echo -e "${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})\n${daterun}" > ${FOLDER}/test.repl yes | cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl else ${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${FOLDER}/test.repl" yes | ${PH_CMD} "cp -rfH ${FOLDER}/test.repl ${REPL_CHECKS[${rcheck}]}/test.repl" fi done fi done if [ "${ST_ACTION}" = "repl" ]; then sleep 5 fi if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo fi fi ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "services" ] || [ "${ST_ACTION}" = "check" ]; then if [ ! -z ${LOCAL_SERVICES+x} ]; then if [ "${ST_ACTION}" != "check" ]; then lip=$(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) uptime=`uptime -p` echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}" DIVIDER false green fi for srvc in "${LOCAL_SERVICES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " fi if [ "$(systemctl is-active ${srvc})" != "active" ]; then if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}; fi touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then if [ ! -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then mv ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime fi toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}; fi touch ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down fi fi else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" fi if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down ]; then if [ -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime ]; then toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime) else toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down) fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NODE_HOSTNAME}" ${STATUS_CHECK_EMAIL}; fi rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.down rm -f ${FOLDER}/${NODE_HOSTNAME}-${srvc}.errtime fi fi done if [ "${ST_ACTION}" != "check" ]; then echo; fi fi ######################## for NTYPE in "${NODE_TYPES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}" DIVIDER; echo -e "${idsST[Reset]}" fi nid=1 var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' fi if [ "${NCMD}" != "" ]; then checkhost=$(CHECK_HOST ${nip}) fi if [ "${checkhost}" != "false" ]; then if [ "${ST_ACTION}" != "check" ]; then uptime=`${NCMD} uptime -p` echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green fi if [ -f ${FOLDER}/${nip}.down ]; then if [ -f ${FOLDER}/${nip}.errtime ]; then toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) else toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) fi rm -f ${FOLDER}/${nip}.down; rm -f ${FOLDER}/${nip}.errtime # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi fi var2=${NTYPE}_SERVICES_CHECK[@] for srvc in "${!var2}"; do NOGOCHK=true; if [ ${srvc} = "gitea" ]; then if [[ $($NCMD /sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *10.5.10.120* ]]; then NOGOCHK=false; fi fi if [ ${NOGOCHK} = true ]; then if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' spc1=${cw}-${#NM_SERVICES[${srvc}]} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " fi if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then if [ ! -f ${FOLDER}/${nip}-${srvc}.down ]; then # if [ "${srvc}" == "gitea" ]; then # if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then # SERVICE gitea stop q # SET-PERMISSIONS gitea q # SERVICE gitea start q # fi # if [ "$(${NCMD} systemctl is-active ${srvc})" != "active" ]; then # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Red]}Not Running - could not fix${idsCL[Default]}" # fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down. Could not fix." | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi # touch ${FOLDER}/${nip}-${srvc}.down # else # if [ "${ST_ACTION}" != "check" ]; then # echo -e "${idsCL[Green]}Running - FIXED${idsCL[Default]}" # fi # if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then # if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) # else # toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) # fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! (!!FIXED!!) It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi # rm -f ${FOLDER}/${nip}-${srvc}.down # rm -f ${FOLDER}/${nip}-${srvc}.errtime # fi # fi # else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi touch ${FOLDER}/${nip}-${srvc}.down # fi else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) if [ $errtime -gt ${RENOTIFY} ]; then if [ ! -f ${FOLDER}/${nip}-${srvc}.errtime ]; then mv ${FOLDER}/${nip}-${srvc}.down ${FOLDER}/${nip}-${srvc}.errtime fi toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}" fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi touch ${FOLDER}/${nip}-${srvc}.down fi fi else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Running${idsCL[Default]}" fi if [ -f ${FOLDER}/${nip}-${srvc}.down ]; then if [ -f ${FOLDER}/${nip}-${srvc}.errtime ]; then toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.errtime) else toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}-${srvc}.down) fi if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_SERVICES[${srvc}]} is back UP! It was down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL}; fi rm -f ${FOLDER}/${nip}-${srvc}.down rm -f ${FOLDER}/${nip}-${srvc}.errtime fi fi fi done else if [ ! -f ${FOLDER}/${nip}.down ]; then touch ${FOLDER}/${nip}.down if [ ! -f ${FOLDER}/${nip}.errtime ]; then touch ${FOLDER}/${nip}.errtime fi # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} # fi else errtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.down) # if [ $errtime -gt ${RENOTIFY} ] && [ "${STATUS_CHECK_EMAIL}" != "" ]; then # echo "${NM_NODETYPES[$NTYPE]}-Node${nid} has been down for $(SHOW_TIME ${toterrtime})" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}" ${STATUS_CHECK_EMAIL} # fi fi if [ "${ST_ACTION}" != "check" ]; then toterrtime=`date +%s`-$(stat -c %Y ${FOLDER}/${nip}.errtime) echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}" fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi nid=`expr $nid + 1` done done fi ######################## ## REPLICATION CHECK ######################## if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}" DIVIDER; echo -e "${idsST[Reset]}" fi for NTYPE in "${NODE_TYPES[@]}"; do GOREPCHK=false; CHR=`date +"%H"`; if [ "${CHR}" -ge "12" ] && [ "${CHR}" -le "13" ] && [ "${NTYPE}" = "NC" ]; then GOREPCHK=true; elif [ "${NTYPE}" != "NC" ]; then GOREPCHK=true; fi if [ "${GOREPCHK}" = "true" ]; then PH=${NTYPE}_HOSTS[0] PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then nid=1 var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e " ${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node1 (${!PH}) <--> ${idsST[Bold]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[Default]}" DIVIDER false green fi checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then if [ -f ${FOLDER}/${nip}.down ]; then rm -f ${FOLDER}/${nip}.down # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is back UP!" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-UP" ${STATUS_CHECK_EMAIL}; fi fi var2=${NTYPE}_REPL_CHECK[@] # timeout=`date --date='20 seconds' +%s` for rcheck in "${!var2}"; do if [ "${ST_ACTION}" != "check" ]; then c=0; cw=22; spc='' spc1=${cw}-${#REPL_DESC[${rcheck}]} # if [ "${rcheck}" = "nextcloud-data" ] || [ "${rcheck}" = "nextcloud-www" ]; then # timeout=`date --date='2 minutes' +%s` # else timeout=`date --date='20 seconds' +%s` # fi until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${REPL_DESC[${rcheck}]}${spc}: " else # if [ "${rcheck}" = "nextcloud-data" ] OR [ "${rcheck}" = "nextcloud-www" ]; then # timeout=`date --date='2 minutes' +%s` # else timeout=`date --date='1 minutes' +%s` # fi fi checked=false until [ "${checked}" = "" ]; do if [ "${PH_CMD}" = "" ]; then checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl` else checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`" fi if [ "`date +%s`" -gt "$timeout" ]; then timeout=true break fi done if [ "${timeout}" = "true" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Timeout${idsCL[Default]}" fi if [ ! -f ${FOLDER}/${nip}-${rcheck}.down ]; then touch ${FOLDER}/${nip}-${rcheck}.down if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi fi else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Good${idsCL[Default]}" fi if [ -f ${FOLDER}/${nip}-${rcheck}.down ]; then rm -f ${FOLDER}/${nip}-${rcheck}.down if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" | mail -s "Repl-Timeout-'${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi fi fi done else if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Red]}Node is Down!${idsCL[Default]}" fi if [ ! -f ${FOLDER}/${nip}.down ]; then touch ${FOLDER}/${nip}.down # if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${NM_NODETYPES[$NTYPE]}-Node${nid} is down" | mail -s "${NM_NODETYPES[$NTYPE]}-Node${nid}-${nip}-DOWN" ${STATUS_CHECK_EMAIL}; fi fi fi if [ "${ST_ACTION}" != "check" ]; then echo; fi fi nid=`expr $nid + 1` done fi fi done fi ########################## # REMOVE REPL CHECK FILES ########################## if [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "repl" ] || [ "${ST_ACTION}" = "check" ] || [ "${ST_ACTION}" = "" ]; then if [ "${ST_ACTION}" != "check" ]; then echo -en "${idsCL[LightCyan]}Cleaning up status checks... ${idsCL[Default]}" fi #NHCMD="ssh root@${MYSQL_HOSTS[0]}" #LBHCMD="ssh root@${LB_HOSTS[0]}" #WHCMD="ssh root@${WEB_HOSTS[0]}" for NTYPE in "${NODE_TYPES[@]}"; do PH=${NTYPE}_HOSTS[0] PH_CMD="ssh root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do ${PH_CMD} rm -f ${FOLDER}/test.repl daterun=`date +%Y-%m-%d-%H-%M-%S` if [ "${PH_CMD}" = "" ]; then rm -f ${FOLDER}/test.repl rm -f ${REPL_CHECKS[${rcheck}]}/test.repl else ${PH_CMD} rm -f ${FOLDER}/test.repl ${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl fi done fi done if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo fi fi ######################## ## FREE SPACE CHECK ######################## #if [ "${ST_ACTION}" = "" ] || [ "${ST_ACTION}" = "report" ] || [ "${ST_ACTION}" = "freespace" ]; then if [ "${ST_ACTION}" = "freespace" ]; then if [ "${ST_ACTION}" != "repl" ] && [ "${ST_ACTION}" != "services" ]; then for NTYPE in "${NODE_TYPES[@]}"; do if [ "${ST_ACTION}" != "check" ]; then echo -e "${idsST[Bold]}"; DIVIDER echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Free Space Scan${idsCL[Default]}" DIVIDER; echo -e "${idsST[Reset]}" fi nid=1 var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else NCMD="ssh root@${nip}"; LH='' fi if [ "${ST_ACTION}" != "check" ]; then echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_NODETYPES[$NTYPE]}-Node${nid} (${nip})${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYello]}${LH}${idsCL[Default]}" DIVIDER false green fi declare -A partitions partitions=`${NCMD} df -hBM | grep -vE '^Filesystem|tmpfs|cdrom' | awk '{ print $5 " " $1 }' | while read output; do echo $output | awk '{ print $2 }'; done;` partitions2=`echo ${partitions[*]}` partitions3=(${partitions2}) for partition in "${partitions3[@]}"; do if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` freespace=`${NCMD} df -hPBM ${partition} | awk '{print $4}' |tail -1|sed 's/M$//g'` if [ "${freespace}" -le "1000" ]; then fs_status='error' fs_status_color='Red' if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${partition} : ${freespace} MB free" | mail -s "Free Space Critical: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi elif [ "${freespace}" -le "5000" ]; then fs_status='warn' fs_status_color='Yellow' if [ "${STATUS_CHECK_EMAIL}" != "" ]; then echo "${partition} : ${freespace} MB free" | mail -s "Free Space Warning: '${NM_NODETYPES[$NTYPE]}-Node${nid}'" ${STATUS_CHECK_EMAIL}; fi else fs_status='' fs_status_color='Green' fi if [ "${ST_ACTION}" != "check" ]; then c=0; cw=20; spc='' spc1=${cw}-${#partition} until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${pname}$spc: " echo -e "${idsCL[${fs_status_color}]}${freespace} MB${idsCL[Default]}" fi fi done nid=`expr $nid + 1` echo done done fi fi ######################## ######################## if [ "${ST_ACTION}" != "check" ]; then echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi fi end=`date +%s` runtime=$((end-start)) echo "runtime: ${runtime}" } NEWCERT(){ echo "" if [ -z ${1+x} ]; then echo -e -n "${idsCL[LightCyan]}Create certificate for what address: ${idsCL[Default]}" read NEW_CERT echo "" else NEW_CERT=${1} fi if [[ ${NEW_CERT} == *","* ]]; then IFS=','; NEW_CERTS=(${NEW_CERT}); unset IFS MAIN_CERT=${NEW_CERTS[0]} else MAIN_CERT=${NEW_CERT} fi if [ "${NEW_CERT}" != "" ]; then echo -e "${idsCL[LightGreen]}Requesting Certificate for '${idsCL[Yellow]}${NEW_CERT}${idsCL[LightGreen]}'...${idsCL[Default]}" echo "" # echo -en "${idsCL[LightYellow]}Stopping other Webservers... ${idsCL[Default]}" # for nip in "${WEB_HOSTS[@]}"; do # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then # # `ssh root@${nip} service nginx stop` # # SERVICE nginx stop ${nip} # fi # done # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" # echo $CERT_DAEMON certonly --webroot -w /var/www/html -d ${NEW_CERT} # $CERT_DAEMON certonly --dry-run --webroot -w /var/www/html -d ${NEW_CERT} if [ -f /etc/letsencrypt/live/${MAIN_CERT}/cert.pem ]; then chown -R root:letsencrypt /etc/letsencrypt chmod -R 6775 /etc/letsencrypt if [ -f /etc/nginx/sites-available/${MAIN_CERT} ]; then rm -f ${FOLDER}/cert-request.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${NEW_CERT}\n${daterun}" > ${FOLDER}/cert-request.lastrun yes | cp -rfH ${FOLDER}/cert-request.lastrun /etc/letsencrypt/cert-request.lastrun DIVIDER true SERVICE nginx reload web fi echo echo -e "${idsCL[Green]}Certificate has been successfully created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Green]}'...${idsCL[Default]}" else echo echo -e "${idsCL[Red]}Certificate could not be created for '${idsCL[Yellow]}${NEW_CERT}${idsCL[Red]}'...${idsCL[Default]}" fi # echo # echo -en "${idsCL[LightYellow]}Starting other Webservers... ${idsCL[Default]}" # for nip in "${WEB_HOSTS[@]}"; do # if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then # `ssh root@${nip} service nginx start` # # SERVICE nginx start ${nip} # fi # done # echo -e "${idsCL[Green]}Completed${idsCL[Default]}" echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi else echo "You havent entered a site address." sleep 1 NEWCERT exit 0 fi } DEL-SSL(){ if [ -z ${1+x} ]; then echo -e -n "${idsCL[LightCyan]}Delete what SSL site address: ${idsCL[Default]}" read DEL_SSL echo "" else DEL_SSL=${1} fi if [ ! -z ${DEL_SSL+x} ] && [ "${DEL_SSL}" != "" ]; then if [ -d /etc/letsencrypt/live/${DEL_SSL} ]; then echo -e "${idsCL[LightRed]}Deleting the SSL certificates for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}'...${idsCL[Default]}" echo "" echo -en "${idsCL[LightRed]}Removing Files and Folders... ${idsCL[Default]}" rm -rf /etc/letsencrypt/archive/${DEL_SSL} rm -rf /etc/letsencrypt/live/${DEL_SSL} rm -f /etc/letsencrypt/renewal/${DEL_SSL}.conf echo -e "${idsCL[Green]}OK${idsCL[Default]}" echo "" else echo -e "${idsCL[LightRed]}The SSL files folder for '${idsCL[Red]}${DEL_SSL}${idsCL[LightRed]}' could not be found.${idsCL[Default]}" exit 1 fi if [ "${timeout}" != "true" ]; then echo -e "${idsCL[Green]}Completed${idsCL[Default]}" fi echo # SERVICE nginx reload echo -e "${idsCL[LightRed]}The SSL certificate has been removed fromt be nodes.${idsCL[Default]}" fi } CERTRENEW(){ echo -en "${idsCL[LightCyan]}Stopping Webserver-Node2...${idsCL[Default]}" ssh root@10.5.10.122 service nginx stop echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" echo echo -e "${idsCL[LightGreen]}Renewing Certificates...${idsCL[Default]}" echo sleep 5 # $CERT_DAEMON --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun $CERT_DAEMON --force-renewal --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun # $CERT_DAEMON --dry-run --preferred-chain "ISRG Root X1" renew --webroot -w /var/www/html 2>&1 | tee ${FOLDER}/cert-renewal.lastrun chown -R root:letsencrypt /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun chmod -R 6775 /etc/letsencrypt 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun DIVIDER true echo -en "${idsCL[LightCyan]}Starting Webserver-Node2 Back up...${idsCL[Default]}" ssh root@10.5.10.122 service nginx start echo -e "${idsCL[LightGreen]} Completed${idsCL[Default]}" echo SERVICE nginx reload web 2>&1 | tee -a ${FOLDER}/cert-renewal.lastrun echo -e "${idsCL[LightGreen]}The certificates have been renewed.${idsCL[Default]}" echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then DIVIDER true ENTER2CONTINUE fi } NIGHTLYRENEW(){ rm -f ${FOLDER}/cert-renewal.lastrun ssh root@10.5.10.122 service nginx stop sleep 5 $CERT_DAEMON renew --webroot -w /var/www/html &>> ${FOLDER}/cert-renewal.lastrun chown -R root:letsencrypt /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun chmod -R 6775 /etc/letsencrypt &>> ${FOLDER}/cert-renewal.lastrun yes | cp -rfH ${FOLDER}/cert-renewal.lastrun /etc/letsencrypt/cert-renewal.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/letsencrypt/cert-renewal.lastrun ssh root@10.5.10.122 service nginx start SERVICE nginx reload web &>> ${FOLDER}/cert-renewal.lastrun } LISTCERTS(){ declare -i cw; declare -i spc1; declare -i c declare -A CHECKCERT_DOMAINS IFS=$'\n' for LINE in `egrep -v '(^#|^$)' ${FOLDER}/ssl-cert-check/ssldomains`; do HOST=${LINE%% *} PORT=${LINE#* } IFS=" " CHECKCERT_DOMAINS[${HOST}]=${PORT} done unset IFS if [ ! -z ${LOCAL_SERVICES+x} ]; then NCMD="ssh root@${MYSQL_HOSTS[0]}" ${NCMD} rm -f /tmp/ssllist ${NCMD} 'for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done' > /tmp/ssllist else NCMD='' rm -f /tmp/ssllist for certdir in /etc/letsencrypt/live/*/ ; do echo $certdir; done > /tmp/ssllist fi if [ -z $action ] || [ "${action}" = "gui" ]; then DIVIDER true fi echo echo -e "${idsCL[LightGreen]}Current Certificates on Node...${idsCL[Default]}" DIVIDER false yellow 120 echo -e "Subject Name Monitored Expires Alternate Subject Names" DIVIDER false yellow 120 cw=30; c=0; spc2=''; until [ $c = ${cw} ]; do spc2="${spc2} "; c=`expr $c + 1`; done for certdir in $( ${FOLDER}/ssl-cert-check.lastrun if [ "${STATUS_CHECK_EMAIL}" != "" ]; then SSLLOG="$(cat ${FOLDER}/ssl-cert-check.lastrun)" echo "SSL Certs Check\n\n$SSLLOG" | mail -s "SSL Certs Check" david@schroedercity.com echo "SSL Certs Check\n\n$SSLLOG" | mail -s "SSL Certs Check" 9208839613@mms.att.net fi else ${FOLDER}/ssl-cert-check/ssl-cert-check -a -f ${FOLDER}/ssl-cert-check/ssldomains fi } BACKUP(){ BACKUP_FOLDER=/opt/idssys/backups/node-backup #BACKUP_FOLDER=/opt/idssys/backups/node-backups/${NODE_HOSTNAME} #/`date +%Y-%m-%d` #BACKUP_TIMEFOLDER=${BACKUP_DAYFOLDER}/`date +%H-%M` if [ "${action}" != "backup" ]; then echo -e "${idsCL[Green]}Backing up Node Settings and Files...${idsCL[Default]}" DIVIDER true fi if [ ! -d ${BACKUP_FOLDER} ]; then mkdir -p ${BACKUP_FOLDER} fi for item in "${!BACKUP_ITEMS[@]}" do if [ "${action}" != "backup" ]; then echo -en "Backing up '${item}'... " fi #tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -C ${BACKUP_ITEMS[$item]} . tar -czPf ${BACKUP_FOLDER}/${item}.tar.gz -g ${BACKUP_FOLDER}/backup.snar -C ${BACKUP_ITEMS[$item]} . --exclude='./data/repository' --exclude='*/.stfolder' --exclude='*/.stversions' --exclude='*/.git' if [ "${action}" != "backup" ]; then echo -e "${idsCL[Green]}Ok${idsCL[Default]}" fi done touch ${FOLDER}/backup.lastrun if [ "${action}" != "backup" ]; then echo "" DIVIDER true fi if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi } DELSITE(){ while [ $# -gt 0 ]; do case "$1" in -site) DEL_SITE=${2};; -ssl) DEL_SSL=${2};; -list) DELSITES; exit 0;; -*) echo "Invalid option: '${1}' requires an argument" 1>&2 echo "" echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" width=33 printf "%-${width}s- %s\n" " -site {FQDN address}" "(*required)" printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" echo "}" exit 1;; esac shift done if [ -z ${DEL_SITE+x} ]; then echo -e -n "${idsCL[LightCyan]}Delete what site address: ${idsCL[Default]}" read DEL_SITE echo "" fi if [[ $DEL_SSL =~ ^[Nn]$ ]]; then DEL_SSL=no elif [[ $DEL_SSL =~ ^[Yy]$ ]]; then DEL_SSL=yes elif [ -z ${DEL_SSL+x} ]; then echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${DEL_SITE}' as well? [y/N]${idsCL[Default]} " read DEL_SSL fi if [ ! -z ${DEL_SITE+x} ] && [ "${DEL_SITE}" != "" ]; then echo -e "${idsCL[LightRed]}Deleting site '${idsCL[Red]}${DEL_SITE^^}${idsCL[LightRed]}'...${idsCL[Default]}" echo "" echo -e "${idsCL[LightRed]}[[Removing Files and Folders]]${idsCL[Default]}" echo -e "${idsCL[LightRed]}-------------------------------------------${idsCL[Default]}" nid=1 for nip in "${WEB_HOSTS[@]}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi echo -en "Removing from Webserver-Node${nid} ($nip)... ${idsCL[Default]}" if [ "${NCMD}" != "" ]; then checkhost=$(CHECK_HOST ${nip}) fi if [ "${checkhost}" != "false" ]; then #if [ -f /etc/nginx/sites-available/${DEL_SITE} ]; then ${NCMD} rm -f /etc/nginx/sites-available/${DEL_SITE} ${NCMD} rm -f /etc/nginx/sites-enabled/${DEL_SITE} #fi #if [ -d /var/www/${DEL_SITE} ]; then ${NCMD} rm -rf /var/www/${DEL_SITE} #fi if [ "${DEL_SSL}" = "yes" ]; then ${NCMD} rm -rf /etc/letsencrypt/archive/${DEL_SITE} ${NCMD} rm -rf /etc/letsencrypt/live/${DEL_SITE} ${NCMD} rm -f /etc/letsencrypt/renewal/${DEL_SITE}.conf fi echo -e "${idsCL[Green]}OK${idsCL[Default]}" else echo -e "${idsCL[Red]}Node is down${idsCL[Default]}" fi nid=`expr $nid + 1` done echo "" SERVICE nginx reload ns echo -e "${idsCL[LightRed]}Site has been deleted.${idsCL[Default]}" else echo "Missing arguments" echo "" echo -e "Usage: ${idsCL[Yellow]}nodemgmt delsite${idsCL[Default]} {" width=33 printf "%-${width}s- %s\n" " -site {FQDN address}" "Site to delete" printf "%-${width}s- %s\n" " -ssl {yes or [no]}" "Delete SSL certs as well" printf "%-${width}s- %s\n" " -list" "List sites (same as running nodemgmt delsites)" echo "}" exit 1 fi } DELSITES(){ echo echo -e "${idsCL[Red]}Select a site to delete...${idsCL[Default]}" DIVIDER true sid=1 filels="( $(ssh root@${WEB_HOSTS[0]} ls '/etc/nginx/sites-available/*') )" # IFS='\n' for siteconf in $filels; do # for siteconf in /etc/nginx/sites-available/* ; do # [ -e "$siteconf" ] || continue if [ ${siteconf:0:1} == '/' ]; then IFS='/'; site_conf=(${siteconf}); unset IFS SITES[${sid}]=${site_conf[4]} sid=`expr $sid + 1` fi done for s in "${!SITES[@]}"; do echo " [${s}] ${SITES[${s}]}" done echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then echo " [B] Back" fi echo " [Q] Quit" echo "" echo -e -n "${idsCL[LightYellow]}Please select a site from above from above:${idsCL[Default]} " read selsite echo "" if [ -z ${SITES[$selsite]} ] && [ "${selsite}" != "Q" ] && [ "${selsite}" != "q" ] && [ "${selsite}" != "B" ] && [ "${selsite}" != "b" ]; then echo "Thats an invaild option," echo "please select a valid option only." sleep 1 DELSITES exit 0 elif [ "${selsite}" = "Q" ] || [ "${selsite}" = "q" ]; then exit 0 elif [ "${selsite}" = "B" ] || [ "${selsite}" = "b" ]; then GUI else while : do echo -e -n "${idsCL[LightRed]}Are you sure you want to delete '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}'? [y/N]${idsCL[Default]} " read response echo "" if [[ $response =~ ^[Yy]$ ]]; then echo -e -n "${idsCL[LightRed]}Do you also want to delete the certs for '${idsCL[Red]}${SITES[${selsite}]^^}${idsCL[LightRed]}', if they exist? [y/N]${idsCL[Default]} " read sslresponse DELSITE -site ${SITES[${selsite}]} -ssl ${sslresponse} echo "" DIVIDER ENTER2CONTINUE break else break fi done DELSITES exit 0 fi if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi } NEWSITE(){ echo while [ $# -gt 0 ]; do case "$1" in -site) NEW_SITE=${2};; -type) SITE_TYPE=${2};; -ssl) CREATE_SSL=${2};; -proxy_scheme) PROXYSCHEME=${2};; -proxy_host) PROXYHOST=${2};; -proxy_port) PROXYPORT=${2};; -h | -help | --help) echo "Missing arguments" echo "" echo -e "Usage: ${idsCL[Yellow]}nodemgmt newcert${idsCL[Default]} {" width=33 printf "%-${width}s- %s\n" " -site {FQDN address(,es)}" "(new site and aliases, comma separated)" printf "%-${width}s- %s\n" " -ssl {yes or no}" "(defaults to yes)" printf "%-${width}s- %s\n" " -type {'local' or 'proxy'}" "(defaults to local)" printf "%-${width}s- %s\n" " -proxy_port {host port}" "(proxy backend host)" printf "%-${width}s- %s\n" " -proxy_host {IP or FQDN}" "(proxy backend port)" printf "%-${width}s- %s\n" " -sproxy_cheme {http or https}" "(proxy backend scheme)" echo "}" exit 1;; esac shift done #if [ -z ${SITE_TYPE+x} ]; then SITE_TYPE=local; fi #if [ -z ${CREATE_SSL+x} ]; then CREATE_SSL=true; fi if [ -z ${NEW_SITE+x} ]; then echo -e -n "${idsCL[LightCyan]}New site domain name: ${idsCL[Default]}" read NEW_SITE showdivide=yes fi if [ -z ${CREATE_SSL+x} ]; then echo -e -n "${idsCL[LightCyan]}Create SSL for site? [Y/n] ${idsCL[Default]}" read CREATE_SSL showdivide=yes if [[ $CREATE_SSL =~ ^[Yy]$ ]] || [ "${CREATE_SSL}" = "" ]; then CREATE_SSL=yes else CREATE_SSL=no fi fi if [ -z ${SITE_TYPE+x} ]; then echo -e -n "${idsCL[LightCyan]}Site type (local/{proxy}): ${idsCL[Default]}" read SITE_TYPE showdivide=yes if [ "${SITE_TYPE}" = "" ]; then SITE_TYPE=proxy fi fi if [ "${SITE_TYPE}" = "proxy" ]; then if [ -z ${PROXYHOST+x} ]; then echo -e -n "${idsCL[LightCyan]}What is the proxy backend address (IP or FQDN): ${idsCL[Default]}" read PROXYHOST showdivide=yes fi if [ -z ${PROXYPORT+x} ]; then echo -e -n "${idsCL[LightCyan]}What is the proxy backend port (tcp port): ${idsCL[Default]}" read PROXYPORT showdivide=yes fi if [ -z ${PROXYSCHEME+x} ]; then echo -e -n "${idsCL[LightCyan]}What is the proxy backend scheme (http/https): ${idsCL[Default]}" read PROXYSCHEME showdivide=yes fi fi if [ "${showdivide}" = "yes" ]; then DIVIDER; fi echo "" width=14 printf "%-${width}s: %s\n" "New site" "${NEW_SITE}" printf "%-${width}s: %s\n" "Create SSL" "${CREATE_SSL}" printf "%-${width}s: %s\n" "Site type" "${SITE_TYPE}" if [ "${SITE_TYPE}" = "proxy" ]; then printf "%-${width}s: %s\n" "Proxy host" "${PROXYHOST}" printf "%-${width}s: %s\n" "Proxy port" "${PROXYPORT}" printf "%-${width}s: %s\n" "Proxy scheme" "${PROXYSCHEME}" fi echo -e -n "${idsCL[LightRed]}Is this information correct? [Y/n]${idsCL[Default]} " read response echo if [[ $response =~ ^[Yy]$ ]] || [ "${response}" = "" ]; then if [[ ${NEW_SITE} == *","* ]]; then IFS=','; NEW_SITES=(${NEW_SITE}); unset IFS MAIN_SITE=${NEW_SITES[0]} NGINX_SERVERNAME=${NEW_SITE//[,]/ } else MAIN_SITE=${NEW_SITE} NGINX_SERVERNAME=${NEW_SITE} fi if [ "${SITE_TYPE}" = "proxy" ]; then if [ ! -z ${PROXYSCHEME+x} ] && [ ! -z ${PROXYHOST+x} ] && [ ! -z ${PROXYPORT+x} ]; then GO=true; fi else GO=true fi if [ "${GO}" = "true" ]; then echo -e "${idsCL[LightGreen]}Setting up new site for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}" echo "" if [ "${SITE_TYPE}" = "local" ]; then echo -e "server { listen 80;" > /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " server_name ${NGINX_SERVERNAME}; set \$base /var/www/${MAIN_SITE}; root \$base/public_html; access_log /var/log/nginx/${MAIN_SITE}-access.log; error_log /var/log/nginx/${MAIN_SITE}-error.log warn;" >> /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem; include conf.d/include/ssl-ciphers.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " index index.php; location / { try_files \$uri \$uri/ /index.php?\$query_string;" >> /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " } location ~ \.php\$ { fastcgi_pass unix:/var/run/php/php7.2-fpm.sock; include conf.d/include/php_fastcgi.conf; } include conf.d/include/general.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " include conf.d/include/letsencrypt-acme-challenge.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e "}" >> /etc/nginx/sites-available/${MAIN_SITE} mkdir -p /var/www/${MAIN_SITE}/{public_html,nginx_logs} echo -en "${idsCL[LightYellow]}Waiting for folder replication across the webserver nodes... ${idsCL[Default]}" for nip in "${WEB_HOSTS[@]}"; do checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) != *"${nip}"* ]]; then checked=false until [ "${checked}" = "" ]; do checked=`ssh root@${nip} "[ ! -d /var/www/${MAIN_SITE} ] && echo does not exist"` done fi fi done echo -e "${idsCL[Green]}Completed${idsCL[Default]}" echo -en "${idsCL[LightYellow]}Setting folder permissions... ${idsCL[Default]}" SET-PERMISSIONS ${MAIN_SITE} echo -e "${idsCL[Green]}Completed${idsCL[Default]}" else echo -e "server { set \$forward_scheme ${PROXYSCHEME}; set \$server \"${PROXYHOST}\"; set \$port ${PROXYPORT}; listen 80;" > /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " listen 443 ssl http2;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " server_name ${NGINX_SERVERNAME};" >> /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " include conf.d/include/letsencrypt-acme-challenge.conf; include conf.d/include/ssl-ciphers.conf; ssl_certificate /etc/letsencrypt/live/${MAIN_SITE}/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/${MAIN_SITE}/privkey.pem;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " access_log /var/log/nginx/proxy-${MAIN_SITE}.log proxy; location / {" >> /etc/nginx/sites-available/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then echo -e " include conf.d/include/force-ssl.conf;" >> /etc/nginx/sites-available/${MAIN_SITE} fi echo -e " include conf.d/include/proxy.conf; } } " >> /etc/nginx/sites-available/${MAIN_SITE} fi ln -s /etc/nginx/sites-available/${MAIN_SITE} /etc/nginx/sites-enabled/${MAIN_SITE} if [ "${CREATE_SSL}" = "yes" ]; then NEWCERT ${NEW_SITE} fi rm -f ${FOLDER}/new-site.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${NEW_SITE}\n${daterun}" > ${FOLDER}/new-site.lastrun yes | cp -rfH ${FOLDER}/new-site.lastrun /etc/nginx/new-site.lastrun yes | cp -rfH ${FOLDER}/new-site.lastrun /var/www/new-site.lastrun daterun=`date +%Y-%m-%d-%H-%M-%S` echo -e "${daterun}" >> /etc/nginx/new-site.lastrun DIVIDER true SERVICE nginx reload web echo "" echo -e "${idsCL[LightGreen]}The new site for '${idsCL[LightGreen]}${NEW_SITE}${idsCL[Default]}' has been created.${idsCL[Default]}" echo "" else echo "Missing proxy arguments" Exit 1 fi else ${SCRIPT} newsite exit 0 fi } SET-PERMISSIONS(){ if [ "${1}" = "gitea" ]; then nid=1 for nip in "${WEB_HOSTS[@]}"; do if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' else NCMD="ssh root@${nip}" fi if [ "${2}" != "q" ]; then echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on Webserver-Node${nid} (${nip})... ${idsCL[Default]}" fi checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then ${NCMD} chown -R git:git /var/lib/gitea ${NCMD} chmod -R 750 /var/lib/gitea ${NCMD} chown -R git:git /home/git ${NCMD} chmod -R 750 /home/git if [ "${2}" != "q" ]; then echo -e "${idsCL[Green]}Complete${idsCL[Default]}" fi elif [ "${2}" != "q" ]; then echo -e "${idsCL[Red]}Node is down${idsCL[Default]}" fi nid=`expr $nid + 1` done elif [ "${1}" = "nextcloud" ]; then nip=${NC_HOSTS[0]} if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then NCMD='' else NCMD="ssh root@${nip}" fi if [ "${2}" != "q" ]; then echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on Nextcloud-Node1 (${nip})... ${idsCL[Default]}" fi checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then ${NCMD} chown -R www-data:www-data /mnt/Nextcloud-Data ${NCMD} find /mnt/Nextcloud-Data/ -type d -exec chmod 750 {} \; ${NCMD} find /mnt/Nextcloud-Data/ -type f -exec chmod 640 {} \; if [ "${2}" != "q" ]; then echo -e "${idsCL[Green]}Complete${idsCL[Default]}" fi elif [ "${2}" != "q" ]; then echo -e "${idsCL[Red]}Node is down${idsCL[Default]}" fi else DIVIDER if [ "${1}" != "" ]; then NEW_SITE=${1} echo -e "${idsCL[Yellow]}Setting new site folder permissions for (/var/www/${NEW_SITE})${idsCL[Default]}" fi if [ "${NEW_SITE}" = "xxx" ]; then nid=1 for nip in "${MYSQL_HOSTS[@]}"; do checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi echo -e "${idsCL[Yellow]}Setting folder permissions for MySQL-Node${nid} (${nip})${idsCL[Default]}" echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}" ${NCMD} chown -R root:letsencrypt /etc/letsencrypt ${NCMD} chmod -R 6775 /etc/letsencrypt echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo "" fi nid=`expr $nid + 1` done fi nid=1 for nip in "${WEB_HOSTS[@]}"; do checkhost=$(CHECK_HOST ${nip}) if [ "${checkhost}" != "false" ]; then if [[ $(/sbin/ip -o -4 addr list eth0 | awk '{print $4}' | cut -d/ -f1) == *"${nip}"* ]]; then nip='localhost ' NCMD='' else NCMD="ssh root@${nip}" fi if [ "${NEW_SITE}" != "" ]; then echo -en "${idsCL[Cyan]}Webserver-Node${nid} (${nip}) ${idsCL[Default]}" ${NCMD} chown -R www-data:www-data /var/www/$NEW_SITE ${NCMD} chmod -R 7775 /var/www/$NEW_SITE ${NCMD} chown -R www-data:www-data /var/lib/php/sessions ${NCMD} chmod -R 7777 /var/lib/php/sessions echo -e "${idsCL[Green]}Complete${idsCL[Default]}" else echo -e "${idsCL[Yellow]}Setting folder permissions for Webserver-Node${nid} (${nip})${idsCL[Default]}" echo -en "${idsCL[Cyan]}Webserver folders ${idsCL[Default]}" ${NCMD} chown -R www-data:www-data /var/www ${NCMD} chmod -R 7775 /var/www ${NCMD} chown -R www-data:www-data /var/lib/php/sessions ${NCMD} chmod -R 7777 /var/lib/php/sessions echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo -en "${idsCL[Cyan]}Gitea Folder ${idsCL[Default]}" ${NCMD} chown -R git:git /var/lib/gitea ${NCMD} chmod -R 750 /var/lib/gitea ${NCMD} chown -R git:git /home/git ${NCMD} chmod -R 750 /home/git echo -e "${idsCL[Green]}Complete${idsCL[Def ault]}" echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}" ${NCMD} chown -R root:letsencrypt /etc/letsencrypt ${NCMD} chmod -R 6775 /etc/letsencrypt echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo -en "${idsCL[Cyan]}PowerDNS-Admin ${idsCL[Default]}" ${NCMD} chown -R www-data:www-data /opt/powerdns-admin ${NCMD} chmod -R 775 /opt/powerdns-admin echo -e "${idsCL[Green]}Complete${idsCL[Default]}" echo "" fi fi nid=`expr $nid + 1` done if [ "${NEW_SITE}" != "" ]; then echo ""; fi if [ -z $action ] || [ "${action}" = "gui" ]; then ENTER2CONTINUE fi fi } SERVICES(){ while : do echo "" echo -e "${idsCL[Green]}Node Services${idsCL[Default]}" DIVIDER true s=1 for srvc in "${NODE_SERVICES[@]}" do echo " [${s}] ${NM_SERVICES[${srvc}]}" s=`expr $s + 1` done echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then echo " [B] Back" fi echo " [Q] Quit" echo "" echo -e -n "${idsCL[LightYellow]}Please select a service from above:${idsCL[Default]} " read -n 1 selsrvc selsrvc=`expr $selsrvc - 1` echo "" if [ -z ${NODE_SERVICES[${selsrvc}]} ] && [ "${selsrvc}" != "Q" ] && [ "${selsrvc}" != "q" ] && [ "${selsrvc}" != "B" ] && [ "${selsrvc}" != "b" ]; then echo "Thats an invaild option," echo "please select a valid option only." sleep 1 SERVICES exit 0 elif [ "${selsrvc}" = "Q" ] || [ "${selsrvc}" = "q" ]; then exit 0 elif [ "${selsrvc}" = "B" ] || [ "${selsrvc}" = "b" ]; then GUI else echo "" echo -e "${idsCL[Green]}Node Service Selected: '${idsST[Bold]}${NM_SERVICES[${NODE_SERVICES[${selsrvc}]}]}${idsST[Reset]}${idsCL[Green]}'${idsCL[Default]}" DIVIDER true s=1 for srvcopt in "${NM_SRVCOPT[@]}"; do echo " [${s}] ${srvcopt~}" s=`expr $s + 1` done echo "" if [ -z $action ] || [ "${action}" = "gui" ]; then echo " [B] Back" fi echo " [Q] Quit" echo "" echo -e -n "${idsCL[LightYellow]}Please select an action to run from above:${idsCL[Default]} " read -n 1 selsrvcopt selsrvcopt=`expr $selsrvcopt - 1` echo "" if [ -z ${NM_SRVCOPT[${selsrvcopt}]} ] && [ "${selsrvcopt}" != "Q" ] && [ "${selsrvcopt}" != "q" ] && [ "${sels\x72\x76\x63\x6F\x70\x74\x7D\x22\x20\x21\x3D\x20\x22\x42\x22\x20\x5D\x20\x26\x26\x20\x5B\x20\x22\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x6F\x70\x74\x7D\x22\x20\x21\x3D\x20\x22\x62\x22\x20\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x09\x09\x65\x63\x68\x6F\x20\x22\x54\x68\x61\x74\x73\x20\x61\x6E\x20\x69\x6E\x76\x61\x69\x6C\x64\x20\x6F\x70\x74\x69\x6F\x6E\x2C\x22\x0A\x09\x09\x09\x09\x65\x63\x68\x6F\x20\x22\x70\x6C\x65\x61\x73\x65\x20\x73\x65\x6C\x65\x63\x74\x20\x61\x20\x76\x61\x6C\x69\x64\x20\x6F\x70\x74\x69\x6F\x6E\x20\x6F\x6E\x6C\x79\x2E\x22\x0A\x09\x09\x09\x09\x73\x6C\x65\x65\x70\x20\x31\x0A\x09\x09\x09\x09\x53\x45\x52\x56\x49\x43\x45\x53\x0A\x09\x09\x09\x09\x65\x78\x69\x74\x20\x30\x0A\x09\x09\x09\x65\x6C\x69\x66\x20\x5B\x20\x22\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x7D\x22\x20\x3D\x20\x22\x51\x22\x20\x5D\x20\x7C\x7C\x20\x5B\x20\x22\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x7D\x22\x20\x3D\x20\x22\x71\x22\x20\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x09\x09\x65\x78\x69\x74\x20\x30\x0A\x09\x09\x09\x65\x6C\x69\x66\x20\x5B\x20\x22\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x7D\x22\x20\x3D\x20\x22\x42\x22\x20\x5D\x20\x7C\x7C\x20\x5B\x20\x22\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x7D\x22\x20\x3D\x20\x22\x62\x22\x20\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x09\x09\x53\x45\x52\x56\x49\x43\x45\x53\x0A\x09\x09\x09\x09\x65\x78\x69\x74\x20\x30\x0A\x09\x09\x09\x65\x6C\x73\x65\x0A\x09\x09\x09\x09\x65\x63\x68\x6F\x0A\x09\x09\x09\x09\x53\x45\x52\x56\x49\x43\x45\x20\x24\x7B\x4E\x4F\x44\x45\x5F\x53\x45\x52\x56\x49\x43\x45\x53\x5B\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x7D\x5D\x7D\x20\x24\x7B\x4E\x4D\x5F\x53\x52\x56\x43\x4F\x50\x54\x5B\x24\x7B\x73\x65\x6C\x73\x72\x76\x63\x6F\x70\x74\x7D\x5D\x7D\x0A\x09\x09\x09\x09\x65\x63\x68\x6F\x20\x22\x22\x0A\x09\x09\x09\x09\x69\x66\x20\x5B\x20\x2D\x7A\x20\x24\x61\x63\x74\x69\x6F\x6E\x20\x5D\x20\x7C\x7C\x20\x5B\x20\x22\x24\x7B\x61\x63\x74\x69\x6F\x6E\x7D\x22\x20\x3D\x20\x22\x67\x75\x69\x22\x20\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x09\x09\x09\x44\x49\x56\x49\x44\x45\x52\x0A\x09\x09\x09\x09\x09\x45\x4E\x54\x45\x52\x32\x43\x4F\x4E\x54\x49\x4E\x55\x45\x0A\x09\x09\x09\x09\x66\x69\x0A\x09\x09\x09\x09\x62\x72\x65\x61\x6B\x0A\x09\x09\x09\x66\x69\x0A\x09\x09\x66\x69\x0A\x09\x64\x6F\x6E\x65\x0A\x09\x0A\x7D\x0A\x0A\x53\x45\x52\x56\x49\x43\x45\x28\x29\x7B\x0A\x09\x4E\x4F\x44\x45\x5F\x54\x59\x50\x45\x3D\x27\x27\x0A\x09\x23\x76\x61\x72\x3D\x4E\x43\x5F\x48\x4F\x53\x54\x53\x5B\x40\x5D\x0A\x09\x66\x6F\x72\x20\x4E\x54\x59\x50\x45\x20\x69\x6E\x20\x22\x24\x7B\x4E\x4F\x44\x45\x5F\x54\x59\x50\x45\x53\x5B\x40\x5D\x7D\x22\x3B\x20\x64\x6F\x0A\x09\x09\x76\x61\x72\x3D\x24\x7B\x4E\x54\x59\x50\x45\x7D\x5F\x48\x4F\x53\x54\x53\x5B\x40\x5D\x0A\x09\x09\x66\x6F\x72\x20\x6E\x69\x70\x20\x69\x6E\x20\x22\x24\x7B\x21\x76\x61\x72\x7D\x22\x3B\x20\x64\x6F\x0A\x09\x09\x09\x49\x50\x3D\x24\x28\x2F\x73\x62\x69\x6E\x2F\x69\x70\x20\x2D\x6F\x20\x2D\x34\x20\x61\x64\x64\x72\x20\x6C\x69\x73\x74\x20\x65\x74\x68\x30\x20\x7C\x20\x61\x77\x6B\x20\x27\x7B\x70\x72\x69\x6E\x74\x20\x24\x34\x7D\x27\x20\x7C\x20\x63\x75\x74\x20\x2D\x64\x2F\x20\x2D\x66\x31\x29\x0A\x09\x09\x09\x69\x66\x20\x5B\x5B\x20\x24\x28\x2F\x73\x62\x69\x6E\x2F\x69\x70\x20\x2D\x6F\x20\x2D\x34\x20\x61\x64\x64\x72\x20\x6C\x69\x73\x74\x20\x65\x74\x68\x30\x20\x7C\x20\x61\x77\x6B\x20\x27\x7B\x70\x72\x69\x6E\x74\x20\x24\x34\x7D\x27\x20\x7C\x20\x63\x75\x74\x20\x2D\x64\x2F\x20\x2D\x66\x31\x29\x20\x3D\x3D\x20\x2A\x22\x24\x7B\x6E\x69\x70\x7D\x22\x2A\x20\x5D\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x09\x09\x4E\x4F\x44\x45\x5F\x54\x59\x50\x45\x3D\x24\x7B\x4E\x54\x59\x50\x45\x7D\x3B\x0A\x09\x09\x09\x09\x62\x72\x65\x61\x6B\x20\x32\x0A\x09\x09\x09\x66\x69\x0A\x09\x09\x64\x6F\x6E\x65\x0A\x09\x64\x6F\x6E\x65\x0A\x09\x69\x66\x20\x5B\x20\x22\x24\x4E\x4F\x44\x45\x5F\x54\x59\x50\x45\x22\x20\x3D\x20\x22\x4E\x43\x22\x20\x5D\x3B\x20\x74\x68\x65\x6E\x0A\x09\x09\x4E\x54\x5F\x48\x4F\x53\x54\x53\x3D\x4E\x43\x5F\x48\x4F\x53\x54\x53\x5B\x40\x5D\x0A\x09\x09\x4E\x54\x5F\x48\x4F\x53\x54\x3D\x4E\x43\x5F\x48\x4F\x53\x54\x53\x5B\x30\x5D\x0A\x09\x09\x4E\x54\x53\x3D\x28\x27\x4E\x43\x27\x29\x3B\x20\x23\x4E\x54\x53\x3D\x24\x4E\x54\x53\x31\x5B\x40\x5D\x0A\x09\x6