voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-03 23:05:23 -05:00
parent d02c6ccf36
commit c1fdda6e7d
4 changed files with 173 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
inc/status.inc
View File

@@ -24,10 +24,10 @@ STATUS(){
var=${NTYPE}_REPL_CHECK[@]
if [ ! -z ${!var+x} ]; then
for rcheck in "${!var}"; do
# echo "CHECK: ${NTYPE} - ${!PH} - ${REPL_CHECKS[${rcheck}]}"
# echo "CHECK: ${NTYPE} - ${!PH} - ${NM_REPL_CHECK_LOC[${rcheck}]}"
${PH_CMD} rm -f ${FOLDER}/test.repl
daterun=`date +%Y-%m-%d-%H-%M-%S`
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${REPL_CHECKS[${rcheck}]}/test.repl" &
${PH_CMD} "echo -e \"Status-Check (${NODE_HOSTNAME})\n${daterun}\" > ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" &
done
fi
fi
@@ -119,12 +119,12 @@ STATUS(){
for NTYPE in "${NODE_TYPES[@]}"; do
nid=1
dockers=${NTYPE}_DOCKER[@]
dockers=${NTYPE}_DOCKERS_CHECK[@]
hosts=${NTYPE}_HOSTS[@]
var=${NTYPE}_HOSTS[@]
if [[ ! -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" == "dockers" ]; then
if [[ ! -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]; then
GOFORCHECK=false;
else
GOFORCHECK=true;
@@ -323,7 +323,7 @@ STATUS(){
########################
## DOCKER CHECK
########################
if [[ -v ${NTYPE}_DOCKER ]] && [ "${ST_ACTION}" != "services" ]; then
if [[ -v ${NTYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo
echo -e "${idsCL[Yellow]} Docker Service(s) Status${idsCL[Default]}"
@@ -336,39 +336,39 @@ STATUS(){
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''
spc1=`expr ${cw} - ${#NM_DOCKERS[${docker}]}`
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${NM_DOCKERS[${docker}]}$spc: "
echo -en " ${NM_DOCKER_DESC[${docker}]}$spc: "
fi
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
${NCMD} docker start ${docker} >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not running, starting now" >> ${LOGFILE}
else
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
${NCMD} /usr/local/bin/docker-compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d >/dev/null 2>&1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is not found, creating and starting now" >> ${LOGFILE}
fi
sleep 10s
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
fi
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is fixed"
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is fixed"
rm -f ${TMPFOLDER}/${nip}~${docker}.*
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is fixed" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${LOGFILE}
else
if [ "${ST_ACTION}" != "check" ]; then
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${docker}.down ]; then
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKERS[${docker}]} is down, could not fix" 1
SENDNOTICE "${NM_HOSTNAMES[${nip}]}-${nip}" "${NM_DOCKER_DESC[${docker}]} is down, could not fix" 1
touch ${TMPFOLDER}/${nip}~${docker}.down
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKERS[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - ${NM_DOCKER_DESC[${docker}]} (docker) is down, could not fix" >> ${LOGFILE}
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}~${docker}.down)
if [ $errtime -gt ${RENOTIFY} ]; then
@@ -482,11 +482,11 @@ STATUS(){
for rcheck in "${!var2}"; do
if [ "${ST_ACTION}" != "check" ]; then
if [ "${ST_ACTION}" != "report" ]; then
c=0; cw=30; spc=''; spc1=${cw}-${#REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
c=0; cw=30; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
else
spc=' '
fi
echo -en " ${REPL_DESC[${rcheck}]}${spc}: "
echo -en " ${NM_REPL_DESC[${rcheck}]}${spc}: "
fi
@@ -504,15 +504,15 @@ STATUS(){
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
if [ ! -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
touch ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})" 1
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder timeout, it is not syncing" >> ${LOGFILE}
fi
else
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Green]}Good${idsCL[Default]}"
if [ -f ${TMPFOLDER}/${nip}~${rcheck}.down ]; then
rm -f ${TMPFOLDER}/${nip}~${rcheck}.down
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${REPL_DESC[${rcheck}]} (${REPL_CHECKS[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
SENDNOTICE "Repl-Timeout-'${NM_HOSTNAMES[${nip}]}'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})"
echo "$(date) - ${nip} - ${NM_HOSTNAMES[${nip}]} - (${NM_REPL_DESC[${rcheck}]}) Replicated folder is back up" >> ${LOGFILE}
fi
fi
done
@@ -548,10 +548,10 @@ STATUS(){
daterun=`date +%Y-%m-%d-%H-%M-%S`
if [ "${PH_CMD}" = "" ]; then
rm -f ${FOLDER}/test.repl &
rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
else
${PH_CMD} rm -f ${FOLDER}/test.repl &
${PH_CMD} rm -f ${REPL_CHECKS[${rcheck}]}/test.repl &
${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl &
fi
done
fi
@@ -704,9 +704,9 @@ REPLCHECK(){
checked=false
until [ "${checked}" == "" ]; do
if [ "${PH_CMD}" == "" ]; then
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${REPL_CHECKS[${rcheck}]}/test.repl" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl`
ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked=`ssh root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl`
else
ssh -q root@${nip} [[ -f ${REPL_CHECKS[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${REPL_CHECKS[${rcheck}]}/test.repl\" | diff - ${REPL_CHECKS[${rcheck}]}/test.repl\"`"
ssh -q root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl ]] && checked="`${PH_CMD} \"ssh root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/test.repl\"`"
fi
[ "`date +%s`" -gt "${timeout}" ] && timeout=true && break
done