voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2025-06-01 12:29:55 -05:00
parent 6f6b272f06
commit caec4404a6
7 changed files with 96 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
defaults.inc
View File

@@ -1,10 +1,11 @@
#!/usr/bin/env bash
VERS='5.5.9-05302025'
VERS='5.5.10-06012025'
NM_BETA=false
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix '
CERT_DAEMON='/opt/certbot/bin/certbot'
SSHCMD='/bin/ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1'
declare -i errtime
[ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt

4
inc/certs.inc
View File

@@ -223,7 +223,7 @@ LISTCERTS(){
fi
# if [ ! -z ${LOCAL_SERVICES+x} ]; then
# NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_HOSTS['LB'][0]}"
# NCMD="${SSHCMD} root@${NM_HOSTS['LB'][0]}"
# #${NCMD} rm -f /tmp/ssllist
# #${NCMD} 'for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done' > /tmp/ssllist
# else
@@ -492,7 +492,7 @@ LISTCERTS_NPM(){
done
unset IFS
if [ ! -z ${LOCAL_SERVICES+x} ]; then
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${MYSQL_HOSTS[0]}"
NCMD="${SSHCMD} root@${MYSQL_HOSTS[0]}"
${NCMD} rm -f /tmp/ssllist
${NCMD} 'for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done' > /tmp/ssllist
else

18
inc/services.inc
View File

@@ -30,8 +30,8 @@ SINGLE_SERVER_SERVICE_CHECK(){
for docker in "${!dockerstocheck}"; do
for nip in "${!hosts}"; do
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
[ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1
[ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1
fi
done
done
@@ -161,7 +161,7 @@ SERVICE_MGMT(){
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
[ ${QUIET} -eq 0 ] && echo -en "${NM_SRVCOPTS[${SERVICE['action']}]}ing on '$nip'... ${idsCL[Default]}"
@@ -195,7 +195,7 @@ SERVICE_MGMT(){
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
if [ ${QUIET} -eq 0 ]; then
@@ -276,7 +276,7 @@ SERVICE_MGMT(){
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD=""
else
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!NT_HOST}"
PH_CMD="${SSHCMD} root@${!NT_HOST}"
fi
${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl
# ${PH_CMD} "echo -e \"Service ${SERVICE['service']} ${SERVICE['action']}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl"
@@ -285,7 +285,7 @@ SERVICE_MGMT(){
if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then
checked=false
until [ "${checked}" = "true" ]; do
checked="`${PH_CMD} ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl)) -le 90 ] && echo true || echo false`"
checked="`${PH_CMD} ${SSHCMD} root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl)) -le 90 ] && echo true || echo false`"
done
fi
done
@@ -310,7 +310,7 @@ SERVICE_MGMT(){
if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then
PH_CMD=""
else
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!NT_HOST}"
PH_CMD="${SSHCMD} root@${!NT_HOST}"
fi
${PH_CMD} touch ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl
for nip in "${!NT_HOSTS}"; do
@@ -363,7 +363,7 @@ SERVICE_MGMT(){
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
NOGOCHK=true;
@@ -386,7 +386,7 @@ SERVICE_MGMT(){
if [ "${SERVICE['service']}"= = "offsite-power-check" ]; then
if [ ${QUIET} -eq 0 ] && [ "${SERVICE['action']}" != "status" ]; then
if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"

30
inc/sites.inc
View File

@@ -54,7 +54,7 @@ DELSITE(){
echo
fi
[ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${DEL_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
[ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${DEL_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
SERVICE_MGMT nginx restart
@@ -139,7 +139,7 @@ NEWSITE(){
exit 0
elif [[ ${overwrite} =~ ^[Yy]$ ]]; then
rm -f ${nginxconfig} >/dev/null 2>&1
[ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${MAIN_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
[ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${MAIN_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
else
exit 0
fi
@@ -282,12 +282,12 @@ NEWSITE(){
echo -e "${idsCL[LightGreen]}Configuring Authelia SSO for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}"
if [ "${SECURE}" == "2FA" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
else
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
fi
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1"
fi
######################################### LOCAL
@@ -474,20 +474,20 @@ NEWPROXYSITE_CREATE(){
if [[ "${SECURE}" = *"FA"* ]]; then
echo -en "${idsCL[LightCyan]}Enabling ${SECURE} SSO access ... "
sed -i "s/#include conf.d\/include\/secure-access.conf/include conf.d\/include\/secure-access.conf/g" ${nginxconfig}
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
if [ "${SECURE}" == "2FA" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
else
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
fi
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1"
${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
${SSHCMD} root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1"
echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
echo
else
echo -en "${idsCL[LightCyan]}Disabling SSO access ... "
sed -i "s/include conf.d\/include\/secure-access.conf/#include conf.d\/include\/secure-access.conf/g" ${nginxconfig}
[ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
[ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml
echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
echo
fi
@@ -557,7 +557,7 @@ SITEINFO(){
done
if [ "${onefacline}" == "" ]; then
[ "$(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1)" != "${NM_AUTHELIA_IP}" ] && ACMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP}" || ACMD=""
[ "$(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1)" != "${NM_AUTHELIA_IP}" ] && ACMD="${SSHCMD} root@${NM_AUTHELIA_IP}" || ACMD=""
autheliaconfig=$(${ACMD} cat ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml)
onefacline=$(echo "${autheliaconfig}" | grep -Fn one_factor | sort | tail -n1)
onefacline=${onefacline%%:*}
@@ -627,8 +627,8 @@ SITEINFO(){
sitefacline=${sitefacline%%:*}
if [ "${sitefacline}" == "" ]; then
SECURE='error'
# ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${SITENAME}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
# ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
# ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${SITENAME}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
# ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml"
elif [ ${sitefacline} -lt ${onefacline} ]; then
SECURE='1FA'
elif [ ${sitefacline} -lt ${twofacline} ]; then

55
inc/status.inc
View File

@@ -68,7 +68,7 @@ STATUS(){
fi
if [ ${skip} -eq 0 ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
@@ -237,7 +237,7 @@ STATUS(){
PH=${NODETYPE}_HOSTS[1]
fi
fi
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
@@ -306,7 +306,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
fi
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
fi
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
${SSHCMD} root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
elif [ "${checked}" == "skip" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightCyan]}Skipped${idsCL[Default]}"
@@ -344,7 +344,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
fi
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
if [ "${PH_CMD}" == "" ]; then
@@ -377,7 +377,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.2023*.repl
if [ $(${PH_CMD} ls ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.*.repl 2>/dev/null | wc -l) -gt 0 ]; then
@@ -413,7 +413,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH=''
else NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
@@ -422,7 +422,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
[ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
declare -A partitions
DRIVEINFO=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=$(${SSHCMD} root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
if [ "${2}" != "report" ]; then
echo -en "\e[1A"
@@ -513,7 +513,7 @@ STATUS_NODE(){
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
NCMD=''; LH='- localhost'
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH=''
NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.* ]; then
checkhost=$(CHECK_HOST ${nip})
@@ -617,7 +617,7 @@ STATUS_NODE(){
fi
sr=0
[ "${NODETYPE}" == "MM" ] && mmstop=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi")
[ "${NODETYPE}" == "MM" ] && mmstop=$(${SSHCMD} root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi")
for srvc in "${!srvcs}"; do
if [ "${STATUS_ACTION}" != "report" ]; then
@@ -629,7 +629,7 @@ STATUS_NODE(){
fi
tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]}
SSCHK=true
if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SSCHK=skip
elif [[ ${tmp[@]} =~ ${srvc} ]]; then
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
@@ -658,7 +658,7 @@ STATUS_NODE(){
mysqlgo=true
fi
else
if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
srvctest=active
else
srvctest=notactive
@@ -684,16 +684,33 @@ STATUS_NODE(){
touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
fi
fi
#########################################
############ SERVICE REPAIRS ############
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 120 ]; then
if [ "${srvc}" == "pihole-FTL" ]; then
${SSHCMD} root@${nip} 'pihole -r' & >/dev/null 2>&1
touch ${NM_TMPFOLDER}/${nip}~${srvc}.fix
fi
fi
#########################################
#########################################
fi
else
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - was FIXED! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} was FIXED, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
else
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
fi
fi
fi
if [[ "${NM_SINGLESRVR_SERVICES[${NODETYPE}]}" = *"${srvc}"* ]]; then
for tnip in "${!NTHOSTS}"; do
@@ -741,7 +758,7 @@ STATUS_NODE(){
fi
tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}
SDCHK=true
if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SDCHK=skip
elif ([ "${NM_HOSTNAMES[${nip}]}" == "ESPHome" ] && [ "${docker}" == "portainer_agent" ]); then
SDCHK=skip
@@ -934,9 +951,9 @@ REPLCHECK(){
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl`
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`${SSHCMD} root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl`
else
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`"
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"${SSHCMD} root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`"
fi
if [ "`date +%s`" -gt "${timeout}" ]; then
timeout=true
@@ -969,7 +986,7 @@ REPLSTART(){
fi
if [ ${skip} -eq 0 ] && [ "${!PH}" != "" ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NODETYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
@@ -998,7 +1015,7 @@ REMOVE_CONFLICT_SYNC_FILES(){
for nip in "${!var}"; do
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
echo -e "${idsCL[LightYellow]}Removing all Syncthing 'sync-conflict' files from ${idsCL[LightGreen]}${NM_HOSTNAMES[${nip}]}${idsCL[LightYellow]} [${nip}]${idsCL[Default]}"
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
PH_CMD="${SSHCMD} root@${nip}"
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
echo -en "Removing files from ${idsCL[LightCyan]}${NM_REPL_CHECK_LOC[${rcheck}]}/* ${idsCL[Default]}... "

2
inc/vCenter-SSL.ps1
View File

@@ -242,6 +242,6 @@ else {
Write-Error "Failed to Replace Certificate, Please verify Correct Configuration and Retry"
}
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 $VEEAMHOSTSSH "Rescan-VBREntity -AllHosts"
${SSHCMD} $VEEAMHOSTSSH "Rescan-VBREntity -AllHosts"
}

62
nodemgmt-scripts.sh
View File

@@ -29,10 +29,10 @@ OFFSITE_STATICROUTE_FIX(){
if [ "$(CHECK_HOST_QC 10.10.0.1)" != "true" ]; then
if [ "$(CHECK_HOST_QC ${NM_OFFSITE_GW_IP})" != "false" ]; then
if [ "$(CHECK_HOST_QC 10.5.0.1)" != "false" ]; then
STSVPN_ADAPTER=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r | grep '10.5.0.0/20' | awk '{print $7}')
if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r | grep 10.10.0.0)" = "" ] && [ "${STSVPN_ADAPTER}" != "" ]; then
RUN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r add 10.6.1.0/24 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null)
RUN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r add 10.10.0.0/16 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null)
STSVPN_ADAPTER=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r | grep '10.5.0.0/20' | awk '{print $7}')
if [ "$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r | grep 10.10.0.0)" = "" ] && [ "${STSVPN_ADAPTER}" != "" ]; then
RUN=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r add 10.6.1.0/24 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null)
RUN=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r add 10.10.0.0/16 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null)
if [ "${RUN}" == "" ]; then
SENDNOTICE "Static Route to Homesite Missing" "Re-added the static route onto the Offsite UniFi Gateway"
else
@@ -100,7 +100,7 @@ SET-PERMISSIONS(){
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}"
@@ -123,7 +123,7 @@ SET-PERMISSIONS(){
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}"
@@ -144,7 +144,7 @@ SET-PERMISSIONS(){
if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
if [ "${2}" != "q" ]; then
echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}"
@@ -174,7 +174,7 @@ SET-PERMISSIONS(){
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -e "${idsCL[Yellow]}Setting folder permissions for ${NM_HOSTNAMES[${nip}]} (${nip})${idsCL[Default]}"
echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}"
@@ -193,7 +193,7 @@ SET-PERMISSIONS(){
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
if [ "${NEW_SITE}" != "" ]; then
echo -en "${idsCL[Cyan]}${NM_HOSTNAMES[${nip}]} (${nip}) ${idsCL[Default]}"
@@ -237,7 +237,7 @@ HAPROXY-CONFIG() {
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}"
@@ -256,7 +256,7 @@ HAPROXY-CONFIG() {
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}"
@@ -274,7 +274,7 @@ HAPROXY-CONFIG() {
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}"
@@ -302,7 +302,7 @@ KEEPALIVE-CONFIG() {
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}"
@@ -321,7 +321,7 @@ KEEPALIVE-CONFIG() {
nip='localhost '
NCMD=''
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
fi
echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}"
@@ -588,9 +588,9 @@ BACKUP_OFFSITEPFSENSE(){
WeeklyBackupsToKeep=8
MonthlyBackupsToKeep=3
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} tar -c -f /cf/conf/conf_backup.tar.gz /cf/conf
${SSHCMD} root@${NM_OFFSITE_GW_IP} tar -c -f /cf/conf/conf_backup.tar.gz /cf/conf
scp root@${NM_OFFSITE_GW_IP}:/cf/conf/conf* ${BAKFOLDER}/${newBAKFOLDER}/
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} rm -f /cf/conf/conf_backup.tar.gz
${SSHCMD} root@${NM_OFFSITE_GW_IP} rm -f /cf/conf/conf_backup.tar.gz
cd ${BAKFOLDER}
if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then
@@ -627,7 +627,7 @@ BACKUP_OFFSITEPFSENSE(){
done;
if [ $keep == 0 ]; then
delFolder="V:\\UniFi-Backups\\Offsite-pfSense_Backups\\${folder##*/}"
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
fi
fi
done
@@ -659,10 +659,10 @@ BACKUP_UNIFIUDM(){
WeeklyBackupsToKeep=2
MonthlyBackupsToKeep=1
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} tar -c -f /volume1/data_backup.tar.gz /data
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} tar -c -f /volume1/root_backup.tar.gz /root
${SSHCMD} root@${UNIFIUDM_IP} tar -c -f /volume1/data_backup.tar.gz /data
${SSHCMD} root@${UNIFIUDM_IP} tar -c -f /volume1/root_backup.tar.gz /root
scp root@${UNIFIUDM_IP}:/volume1/*_backup.tar.gz ${BAKFOLDER}/${newBAKFOLDER}/
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} rm -f /volume1/*_backup.tar.gz
${SSHCMD} root@${UNIFIUDM_IP} rm -f /volume1/*_backup.tar.gz
cd ${BAKFOLDER}
if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then
@@ -699,7 +699,7 @@ BACKUP_UNIFIUDM(){
done;
if [ $keep == 0 ]; then
delFolder="D:\\BackupSync\\UniFi-Backups\\UniFi-UDM_Backups\\${folder##*/}"
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
fi
fi
done
@@ -731,12 +731,12 @@ BACKUP_UNIFINVR(){
WeeklyBackupsToKeep=2
MonthlyBackupsToKeep=1
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/data_backup.tar.gz /data
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/root_backup.tar.gz /root
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/unifi_backup.tar.gz /volume1/.srv/unifi-protect/backups /volume1/.srv/unifi-protect/data /volume1/.srv/unifi-protect/dbBackups
${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/data_backup.tar.gz /data
${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/root_backup.tar.gz /root
${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/unifi_backup.tar.gz /volume1/.srv/unifi-protect/backups /volume1/.srv/unifi-protect/data /volume1/.srv/unifi-protect/dbBackups
scp root@${UNIFINVR_IP}:/volume1/*_backup.tar.gz ${BAKFOLDER}/${newBAKFOLDER}/
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} rm -f /volume1/*_backup.tar.gz
${SSHCMD} root@${UNIFINVR_IP} rm -f /volume1/*_backup.tar.gz
cd ${BAKFOLDER}
if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then
@@ -773,7 +773,7 @@ BACKUP_UNIFINVR(){
done;
if [ $keep == 0 ]; then
delFolder="D:\\BackupSync\\UniFi-Backups\\UniFi-NVR_Backups\\${folder##*/}"
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}"
fi
fi
done
@@ -817,7 +817,7 @@ RUN_COMMAND(){
NCMD=''
LH='- localhost'
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
NCMD="${SSHCMD} root@${nip}"
LH=''
fi
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
@@ -1103,7 +1103,7 @@ MAINTENANCE_MODE(){
DIVIDER . lightCyan ${DV_LEN}
fi
PH=${NTYPE}_HOSTS[0]
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var2=${NTYPE}_HOSTS[@]
for nip in "${!hosts}"; do
echo -e " ${idsCL[Green]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
@@ -1144,8 +1144,8 @@ ESXI_ISCSI_STORAGE_RESCAN(){
for NIP in "${ESXIHOSTS[@]}"; do
if [ $(CHECK_HOST ${NIP}) == true ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NIP} esxcli storage core adapter rescan --all &
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NIP} vmkfstools -V &
${SSHCMD} root@${NIP} esxcli storage core adapter rescan --all &
${SSHCMD} root@${NIP} vmkfstools -V &
fi
done
@@ -1173,7 +1173,7 @@ SETUPSSH(){
ssh-copy-id root@${nip}
fi
# ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "nmg setupssh_back"
# ${SSHCMD} root@${nip} "nmg setupssh_back"
# echo -en "\n${idsCL[LightCyan]}Verifying NodeMgmt ... ${idsCL[Default]}"
# NODEUPDATE ${nip} verify "ssh"