voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2025-06-01 12:29:55 -05:00
parent 6f6b272f06
commit caec4404a6
7 changed files with 96 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
inc/status.inc
View File

@@ -68,7 +68,7 @@ STATUS(){
fi
if [ ${skip} -eq 0 ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
@@ -237,7 +237,7 @@ STATUS(){
PH=${NODETYPE}_HOSTS[1]
fi
fi
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
var=${NTYPE}_HOSTS[@]
@@ -306,7 +306,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
fi
rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl*
fi
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
${SSHCMD} root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' &
elif [ "${checked}" == "skip" ]; then
[ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightCyan]}Skipped${idsCL[Default]}"
@@ -344,7 +344,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
fi
fi
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
if [ "${PH_CMD}" == "" ]; then
@@ -377,7 +377,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
for rcheck in "${!var}"; do
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.2023*.repl
if [ $(${PH_CMD} ls ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.*.repl 2>/dev/null | wc -l) -gt 0 ]; then
@@ -413,7 +413,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
var=${NTYPE}_HOSTS[@]
for nip in "${!var}"; do
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
else NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH=''
else NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${STATUS_ACTION}" != "check" ]; then
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
@@ -422,7 +422,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
[ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
declare -A partitions
DRIVEINFO=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=$(${SSHCMD} root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
if [ "${2}" != "report" ]; then
echo -en "\e[1A"
@@ -513,7 +513,7 @@ STATUS_NODE(){
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
NCMD=''; LH='- localhost'
else
NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH=''
NCMD="${SSHCMD} root@${nip}"; LH=''
fi
if [ "${NCMD}" != "" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.* ]; then
checkhost=$(CHECK_HOST ${nip})
@@ -617,7 +617,7 @@ STATUS_NODE(){
fi
sr=0
[ "${NODETYPE}" == "MM" ] && mmstop=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi")
[ "${NODETYPE}" == "MM" ] && mmstop=$(${SSHCMD} root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi")
for srvc in "${!srvcs}"; do
if [ "${STATUS_ACTION}" != "report" ]; then
@@ -629,7 +629,7 @@ STATUS_NODE(){
fi
tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]}
SSCHK=true
if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SSCHK=skip
elif [[ ${tmp[@]} =~ ${srvc} ]]; then
if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then
@@ -658,7 +658,7 @@ STATUS_NODE(){
mysqlgo=true
fi
else
if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then
srvctest=active
else
srvctest=notactive
@@ -684,16 +684,33 @@ STATUS_NODE(){
touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent
fi
fi
#########################################
############ SERVICE REPAIRS ############
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 120 ]; then
if [ "${srvc}" == "pihole-FTL" ]; then
${SSHCMD} root@${nip} 'pihole -r' & >/dev/null 2>&1
touch ${NM_TMPFOLDER}/${nip}~${srvc}.fix
fi
fi
#########################################
#########################################
fi
else
[ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - was FIXED! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} was FIXED, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
else
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})"
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE}
fi
fi
fi
if [[ "${NM_SINGLESRVR_SERVICES[${NODETYPE}]}" = *"${srvc}"* ]]; then
for tnip in "${!NTHOSTS}"; do
@@ -741,7 +758,7 @@ STATUS_NODE(){
fi
tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}
SDCHK=true
if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then
SDCHK=skip
elif ([ "${NM_HOSTNAMES[${nip}]}" == "ESPHome" ] && [ "${docker}" == "portainer_agent" ]); then
SDCHK=skip
@@ -934,9 +951,9 @@ REPLCHECK(){
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl`
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`${SSHCMD} root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl`
else
ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`"
${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"${SSHCMD} root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`"
fi
if [ "`date +%s`" -gt "${timeout}" ]; then
timeout=true
@@ -969,7 +986,7 @@ REPLSTART(){
fi
if [ ${skip} -eq 0 ] && [ "${!PH}" != "" ]; then
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}"
PH_CMD="${SSHCMD} root@${!PH}"
var=${NODETYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
@@ -998,7 +1015,7 @@ REMOVE_CONFLICT_SYNC_FILES(){
for nip in "${!var}"; do
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
echo -e "${idsCL[LightYellow]}Removing all Syncthing 'sync-conflict' files from ${idsCL[LightGreen]}${NM_HOSTNAMES[${nip}]}${idsCL[LightYellow]} [${nip}]${idsCL[Default]}"
PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"
PH_CMD="${SSHCMD} root@${nip}"
var2=${NTYPE}_REPL_CHECK[@]
for rcheck in "${!var2}"; do
echo -en "Removing files from ${idsCL[LightCyan]}${NM_REPL_CHECK_LOC[${rcheck}]}/* ${idsCL[Default]}... "