update
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
VERS='5.1.5-12112023'
|
||||
VERS='5.1.6-12112023'
|
||||
NM_BETA=false
|
||||
|
||||
noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck '
|
||||
|
||||
@@ -458,7 +458,7 @@ CHECK_DOCKER_SERVICES(){
|
||||
hosts=${NTYPE}_HOSTS[@]
|
||||
|
||||
if [[ -v ${NTYPE}_DOCKERS_CHECK ]]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Service(s) Status${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
@@ -472,7 +472,7 @@ CHECK_DOCKER_SERVICES(){
|
||||
checkhost=$(CHECK_HOST ${nip})
|
||||
fi
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}"
|
||||
uptime=`${NCMD} uptime -p`
|
||||
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||
@@ -490,7 +490,7 @@ CHECK_DOCKER_SERVICES(){
|
||||
fi
|
||||
|
||||
for docker in "${!dockers}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
c=0; cw=30; spc=''
|
||||
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -503,16 +503,16 @@ CHECK_DOCKER_SERVICES(){
|
||||
${NCMD} /usr/local/bin/docker compose -f ${NM_DOCKER_COMPOSE_LOC[${docker}]}/docker-compose.yml up -d
|
||||
sleep 10s
|
||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running - Fixed${idsCL[Default]}"
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not Running - Could Not Fix!${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
@@ -526,12 +526,12 @@ CHECK_DOCKER_SERVICES(){
|
||||
else
|
||||
errtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.down))
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime))
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then echo; fi
|
||||
if [ "${NMG_ACTION}" != "check" ]; then echo; fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
154
inc/status.inc
154
inc/status.inc
@@ -4,16 +4,16 @@ STATUS(){
|
||||
log_start=$(date "+%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# if [ "${1}" == "report" ] || [ "${1}" == "" ]; then
|
||||
# ST_ACTION=${1}
|
||||
# NMG_ACTION=${1}
|
||||
# ntypesel=(${NM_NODE_TYPES[@]})
|
||||
# # if [ "${NM_NODETYPES[${ST_ACTION^^}]}" != "" ]; then
|
||||
# # ntss=${ST_ACTION^^}
|
||||
# # if [ "${NM_NODETYPES[${NMG_ACTION^^}]}" != "" ]; then
|
||||
# # ntss=${NMG_ACTION^^}
|
||||
# # ntypesel=(${ntss})
|
||||
# # ST_ACTION=${2}
|
||||
# # NMG_ACTION=${2}
|
||||
# # else
|
||||
# # ntss=""
|
||||
# # ntypesel=(${NM_NODE_TYPES[@]})
|
||||
# # [ "${1}" == "sync" ] && ST_ACTION=repl || ST_ACTION=${1}
|
||||
# # [ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1}
|
||||
# # fi
|
||||
# else
|
||||
# while [ $# -gt 0 ]; do
|
||||
@@ -35,14 +35,14 @@ STATUS(){
|
||||
# shift
|
||||
# done
|
||||
# fi
|
||||
if [ "${NM_NODETYPES[${ST_ACTION^^}]}" != "" ]; then
|
||||
ntss=${ST_ACTION^^}
|
||||
if [ "${NM_NODETYPES[${1^^}]}" != "" ]; then
|
||||
ntss=${1^^}
|
||||
ntypesel=(${ntss})
|
||||
ST_ACTION=${2}
|
||||
NMG_ACTION=${2}
|
||||
else
|
||||
ntss=""
|
||||
ntypesel=(${NM_NODE_TYPES[@]})
|
||||
[ "${1}" == "sync" ] && ST_ACTION=repl || ST_ACTION=${1}
|
||||
[ "${1}" == "sync" ] && NMG_ACTION=repl || NMG_ACTION=${1}
|
||||
fi
|
||||
|
||||
PRI_CW=40
|
||||
@@ -55,15 +55,15 @@ STATUS(){
|
||||
############################
|
||||
|
||||
([ ! -f ${NM_TMPFOLDER}/.replcheck ] || [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/.replcheck)) -ge 300 ]) && REPLRUN=1 || REPLRUN=0
|
||||
# if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
||||
if [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "" ] || ([ "${ST_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
# if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then
|
||||
if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]) || ([ "${ntss}" != "" ] && [ "${NM_REPL_CHECK[${ntss}]}" != "" ]); then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[LightCyan]}Setting up replication checks ... ${idsCL[Default]}"
|
||||
fi
|
||||
for NTYPE in "${ntypesel[@]}"; do
|
||||
REPLSTART ${NTYPE} &
|
||||
done
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
|
||||
echo -en "${idsCL[LightCyan]}Starting processes to collect/monitor replication check : "
|
||||
fi
|
||||
@@ -78,24 +78,24 @@ STATUS(){
|
||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||
var2=${NTYPE}_REPL_CHECK[@]
|
||||
for rcheck in "${!var2}"; do
|
||||
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${ST_ACTION}" & >/dev/null 2>&1
|
||||
REPLCHECK "${rcheck}" "${nip}" "${PH_CMD}" "${NMG_ACTION}" & >/dev/null 2>&1
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
|
||||
fi
|
||||
|
||||
|
||||
if [ "${ST_ACTION}" == "" ] || [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "services" ] || [ "${ST_ACTION}" == "dockers" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "all" ]; then
|
||||
if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "services" ] || [ "${NMG_ACTION}" == "dockers" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "all" ]; then
|
||||
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : "
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]}Starting processes to collect node service & docker info : "
|
||||
for NTYPE in "${ntypesel[@]}"; do
|
||||
STATUS_NODE ${NTYPE} > ${STATUSRUN_TMPFOLDER}/status-check.${NTYPE}.running 2>/dev/null &
|
||||
done
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}\n"
|
||||
|
||||
########################
|
||||
## LOCALHOST CHECK
|
||||
@@ -103,13 +103,13 @@ STATUS(){
|
||||
|
||||
cpu_usage=$(awk '{u=$2+$4; t=$2+$4+$5; if (NR==1){u1=u; t1=t;} else print ($2+$4-u1) * 100 / (t-t1) "%"; }' <(grep 'cpu ' /proc/stat) <(sleep 1;grep 'cpu ' /proc/stat) | sed -e 's/%//g')
|
||||
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${ST_ACTION}" != "dockers" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ ! -z ${LOCAL_SERVICES+x} ] && [ "${NMG_ACTION}" != "dockers" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
lip=${RUN_NODE_IP}
|
||||
echo -e "\n$(DIVIDER)\n${idsCL[Yellow]}${idsST[Bold]} LOCALHOST Service Status${idsST[Reset]}${idsCL[Default]}"
|
||||
echo -e "$(DIVIDER)"
|
||||
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} ${idsST[Reset]}${idsCL[LightCyan]}[${lip}]"
|
||||
if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
|
||||
if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then
|
||||
uptime=`uptime -p`
|
||||
echo -e "${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
|
||||
else
|
||||
@@ -118,15 +118,15 @@ STATUS(){
|
||||
DIVIDER . green
|
||||
|
||||
fi
|
||||
# if [ "${ST_ACTION}" != "check" ]; then
|
||||
# if [ "${NMG_ACTION}" != "check" ]; then
|
||||
# lip=${RUN_NODE_IP}
|
||||
# uptime=`uptime -p`
|
||||
# echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NODE_HOSTNAME} (${lip})${idsST[Reset]}${idsCL[LightCyan]} - ${uptime} - localhost${idsCL[Default]}"
|
||||
# DIVIDER false green
|
||||
# fi
|
||||
for srvc in "${LOCAL_SERVICES[@]}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "report" ]; then
|
||||
c=0; cw=${PRI_CW}; spc=''
|
||||
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -138,7 +138,7 @@ STATUS(){
|
||||
|
||||
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
|
||||
if [ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
||||
SENDNOTICE "${NODE_HOSTNAME}" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
||||
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
|
||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - LOCAL - ${NODE_HOSTNAME} - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
|
||||
@@ -147,12 +147,12 @@ STATUS(){
|
||||
if [ ${errtime} -gt ${NM_RENOTIFY_TIMEOUT} ]; then
|
||||
[ ! -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime
|
||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
touch ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down
|
||||
fi
|
||||
fi
|
||||
else
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Running${idsCL[Default]}"
|
||||
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.down ]; then
|
||||
if [ -f ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime ]; then
|
||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${NODE_HOSTNAME}~${srvc}.errtime))
|
||||
@@ -203,9 +203,9 @@ STATUS(){
|
||||
## REPLICATION CHECK
|
||||
########################
|
||||
|
||||
# if [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "check" ] || [ "${ST_ACTION}" == "" ]; then
|
||||
if [ "${ST_ACTION}" == "repl" ] || [ "${ST_ACTION}" == "" ] || ([ "${ST_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
# if [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "check" ] || [ "${NMG_ACTION}" == "" ]; then
|
||||
if [ "${NMG_ACTION}" == "repl" ] || [ "${NMG_ACTION}" == "" ] || ([ "${NMG_ACTION}" == "report" ] && [ ${REPLRUN} -eq 1 ]); then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} Replication Status Between the Primary and Secondary Nodes${idsCL[Default]}"
|
||||
DIVIDER; echo -en "${idsST[Reset]}"
|
||||
@@ -219,15 +219,15 @@ STATUS(){
|
||||
var=${NTYPE}_HOSTS[@]
|
||||
for nip in "${!var}"; do
|
||||
if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "${nip}" != "${!PH}" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e " ${idsCL[LightCyan]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
fi
|
||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||
var2=${NTYPE}_REPL_CHECK[@]
|
||||
for rcheck in "${!var2}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "report" ]; then
|
||||
c=0; cw=${PRI_CW}; spc=''; spc1=${cw}-${#NM_REPL_DESC[${rcheck}]}; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
else
|
||||
spc=' '
|
||||
@@ -246,7 +246,7 @@ STATUS(){
|
||||
done
|
||||
|
||||
if [ "${checked}" == "timeout" ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[Red]}Timeout${idsCL[Default]}"
|
||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
||||
touch ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down
|
||||
else
|
||||
@@ -261,7 +261,7 @@ It has been down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDE
|
||||
fi
|
||||
|
||||
elif [ "${checked}" == "good" ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Good${idsCL[Default]}"
|
||||
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.down ]; then
|
||||
if [ -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl.sent ]; then
|
||||
SENDNOTICE "Repl-'${NM_HOSTNAMES[${nip}]}[${nip}]'" "Replicated folder is back up!\n${NM_REPL_DESC[${rcheck}]} (${NM_REPL_CHECK_LOC[${rcheck}]})
|
||||
@@ -275,12 +275,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
done
|
||||
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
[ -f ${NM_FOLDER}/${!PH}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${!PH}]} (${!PH}) is offline${idsCL[Default]}"
|
||||
[ -f ${NM_TMPFOLDER}/${nip}.down ] && echo -e "${idsCL[Red]}${NM_HOSTNAMES[${nip}]}[${nip}] is offline${idsCL[Default]}"
|
||||
fi
|
||||
fi
|
||||
[ "${ST_ACTION}" != "check" ] && echo
|
||||
[ "${NMG_ACTION}" != "check" ] && echo
|
||||
|
||||
fi
|
||||
|
||||
@@ -290,7 +290,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
##########################
|
||||
# REMOVE REPL CHECK FILES
|
||||
##########################
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightCyan]} Cleaning up status checks... ${idsCL[Default]}"
|
||||
for NTYPE in "${ntypesel[@]}"; do
|
||||
PH=${NTYPE}_HOSTS[0]
|
||||
if [ ! -f ${NM_FOLDER}/${!PH}.down ]; then
|
||||
@@ -311,16 +311,16 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
done
|
||||
rm -f ${NM_FOLDER}/test.${STATUS_START//-/}.repl
|
||||
touch ${NM_TMPFOLDER}/.replcheck
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[LightGreen]}Complete${idsCL[Default]}\n"
|
||||
fi
|
||||
|
||||
########################
|
||||
## FREE SPACE CHECK
|
||||
########################
|
||||
#if [ "${ST_ACTION}" == "" ] || [ "${ST_ACTION}" == "report" ] || [ "${ST_ACTION}" == "freespace" ]; then
|
||||
if [ "${ST_ACTION}" == "freespace" ]; then
|
||||
#if [ "${NMG_ACTION}" == "" ] || [ "${NMG_ACTION}" == "report" ] || [ "${NMG_ACTION}" == "freespace" ]; then
|
||||
if [ "${NMG_ACTION}" == "freespace" ]; then
|
||||
for NTYPE in "${ntypesel[@]}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en "${idsST[Bold]}"; DIVIDER
|
||||
echo -e "${idsCL[Yellow]} ${NM_NODETYPES[$NTYPE]}-Node(s) Free Space Scan${idsCL[Default]}"
|
||||
DIVIDER; echo -e "${idsST[Reset]}"
|
||||
@@ -330,12 +330,12 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost'
|
||||
else NCMD="ssh -o ConnectTimeout=3 root@${nip}"; LH=''
|
||||
fi
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||
DIVIDER false green
|
||||
fi
|
||||
|
||||
[ "${ST_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}"
|
||||
declare -A partitions
|
||||
DRIVEINFO=$(ssh -o ConnectTimeout=3 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
|
||||
DRIVEINFO=(${DRIVEINFO})
|
||||
@@ -351,7 +351,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
pname=${pname#*vg-}
|
||||
if [ ${DRIVEINFO[$((${ii}+1))]//M/} -gt 1024 ]; then
|
||||
freespace=${DRIVEINFO[$((${ii}+2))]//M/}
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
c=0; cw=20; spc=''
|
||||
spc1=${cw}-${#pname}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -375,7 +375,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
fs_status=''
|
||||
fs_status_color='Green'
|
||||
fi
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "${idsCL[${fs_status_color}]}${fsdsp} ${idsCL[Default]}"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -383,7 +383,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
# if [ "${partition}" != "udev" ] && [ "${partition}" != "/dev/sda1" ]; then
|
||||
# pname=`echo "${partition}" | awk -F'/' ' { print $NF } '` >/dev/null 2>&1
|
||||
# pname=${pname#*vg-}
|
||||
# if [ "${ST_ACTION}" != "check" ]; then
|
||||
# if [ "${NMG_ACTION}" != "check" ]; then
|
||||
# c=0; cw=20; spc=''
|
||||
# spc1=${cw}-${#pname}
|
||||
# until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -408,7 +408,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
# fs_status_color='Green'
|
||||
#
|
||||
# fi
|
||||
# if [ "${ST_ACTION}" != "check" ]; then
|
||||
# if [ "${NMG_ACTION}" != "check" ]; then
|
||||
# if [[ ${freespace} -gt 1000 ]]; then
|
||||
# fsgb=$(bc <<< "scale=2; ${freespace}/1000")
|
||||
# fsdsp="${fsgb} GB"
|
||||
@@ -426,7 +426,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
done
|
||||
fi
|
||||
########################
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo
|
||||
if [ -z ${ACTION} ] || [ "${ACTION}" == "gui" ]; then
|
||||
DIVIDER true
|
||||
@@ -436,7 +436,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${
|
||||
end=`date +%s`
|
||||
runtime=$((end-start))
|
||||
echo -e "Runtime: ${runtime}\n"
|
||||
[ "${ST_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes
|
||||
[ "${NMG_ACTION}" == "report" ] && echo "${log_start}~${runtime}" >> ${NM_LOGFOLDER}/status-check.scantimes
|
||||
}
|
||||
|
||||
STATUS_NODE(){
|
||||
@@ -446,7 +446,7 @@ STATUS_NODE(){
|
||||
|
||||
NTHOSTS=${NODETYPE}_HOSTS[@]
|
||||
|
||||
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${ST_ACTION}" == "services" ]); then
|
||||
if ([[ ! -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" == "dockers" ]) || ([ "${NM_SERVICES_CHECK[${NODETYPE}]}" == "" ] && [ "${NMG_ACTION}" == "services" ]); then
|
||||
GOFORCHECK=false;
|
||||
else
|
||||
GOFORCHECK=true;
|
||||
@@ -454,9 +454,9 @@ STATUS_NODE(){
|
||||
|
||||
if [ ${GOFORCHECK} = true ]; then
|
||||
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsST[Bold]}"; DIVIDER
|
||||
if [ "${ST_ACTION}" == "dockers" ]; then
|
||||
if [ "${NMG_ACTION}" == "dockers" ]; then
|
||||
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Docker Status${idsCL[Default]}"
|
||||
else
|
||||
echo -e "${idsCL[LightYellow]} ${NM_NODETYPES[$NTYPE]}-Node Service Status${idsCL[Default]}"
|
||||
@@ -464,9 +464,9 @@ STATUS_NODE(){
|
||||
DIVIDER . lightYellow; echo -en "${idsST[Reset]}"
|
||||
fi
|
||||
for nip in "${!NTHOSTS}"; do
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[Default]}"
|
||||
[ "${ST_ACTION}" != "report" ] && echo -en " - Verifying Host..."
|
||||
[ "${NMG_ACTION}" != "report" ] && echo -en " - Verifying Host..."
|
||||
fi
|
||||
|
||||
if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then
|
||||
@@ -485,8 +485,8 @@ STATUS_NODE(){
|
||||
[ "${NODETYPE}" != "OFW" ] && cpu_usage=$(ssh -q -o ConnectTimeout=1 -o ConnectionAttempts=1 root@${nip} "/opt/idssys/defaults/get-data.sh cpu-usage") || cpu_usage=""
|
||||
|
||||
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if ([ "${ST_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${ST_ACTION}" != "report" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
if ([ "${NMG_ACTION}" == "report" ] && [ "${2}" == "email" ]) || [ "${NMG_ACTION}" != "report" ]; then
|
||||
if [ "${NODETYPE}" == "OFW" ]; then
|
||||
uptime=$(${NCMD} uptime | awk '{print $3}' | cut -d, -f1)
|
||||
if [ ${#uptime} -ge 6 ]; then
|
||||
@@ -500,7 +500,7 @@ STATUS_NODE(){
|
||||
else
|
||||
uptime=$(${NCMD} uptime -p)
|
||||
fi
|
||||
[ "${ST_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -e "\r\033[K ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}${idsST[Reset]}${idsCL[LightCyan]} [${nip}]${idsCL[Yellow]}[CPU: `IDS_NUMBER_FORMAT ${cpu_usage} 1`'%]${idsCL[White]} - ${uptime}${idsCL[Default]}"
|
||||
else
|
||||
echo -e "${idsCL[Default]}"
|
||||
fi
|
||||
@@ -519,7 +519,7 @@ STATUS_NODE(){
|
||||
########################
|
||||
## SERVICES CHECK
|
||||
########################
|
||||
if [ "${ST_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
|
||||
if [ "${NMG_ACTION}" != "dockers" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ]; then
|
||||
echo -e "${idsCL[Green]} System Service(s) Status${idsCL[Default]}"
|
||||
DIVIDER . green 52
|
||||
|
||||
@@ -528,7 +528,7 @@ STATUS_NODE(){
|
||||
[ "${NODETYPE}" != "OFW" ] && srvctst=(`${NCMD} "systemctl is-active ${srvcstotest}"`)
|
||||
sr=0
|
||||
for srvc in "${!srvcs}"; do
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
if [ "${NMG_ACTION}" != "report" ]; then
|
||||
c=0; cw=${PRI_CW}; spc=''
|
||||
spc1=${cw}-${#NM_SERVICE_DESC[${srvc}]}
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -550,7 +550,7 @@ STATUS_NODE(){
|
||||
[ "${srvc}" == "keepalived" ] && ([ "${nip}" == "10.2.1.2" ] || [ "${nip}" == "10.2.1.51" ]) && SSCHK=false
|
||||
|
||||
if [ ${SSCHK} != false ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_SERVICE_DESC[${srvc}]}${spc}${idsCL[Default]}: "
|
||||
if [ "${NODETYPE}" != "OFW" ]; then
|
||||
srvctest=${srvctst[$sr]}
|
||||
if [ "${srvctest}" != "active" ] && [ "${srvc}" == "mysql" ]; then
|
||||
@@ -569,14 +569,14 @@ STATUS_NODE(){
|
||||
fi
|
||||
if [ "${srvctest}" != "active" ] && [ "${mysqlgo}" != "true" ]; then
|
||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not Running"
|
||||
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
||||
# SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_SERVICE_DESC[${srvc}]} is down" 1
|
||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is down" >> ${NM_LOGFILE}
|
||||
[ "${srvc}" == "pdnsadmin" ] && ${NCMD} systemctl restart ${srvc} &
|
||||
|
||||
elif [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})"
|
||||
if ([ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.sent)) -gt ${NM_RENOTIFY_TIMEOUT} ]) || ([ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 60 ]); then
|
||||
[ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && mv ${NM_TMPFOLDER}/${nip}~${srvc}.down ${NM_TMPFOLDER}/${nip}~${srvc}.errtime
|
||||
touch ${NM_TMPFOLDER}/${nip}~${srvc}.down
|
||||
@@ -585,7 +585,7 @@ STATUS_NODE(){
|
||||
fi
|
||||
fi
|
||||
else
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running"
|
||||
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then
|
||||
if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then
|
||||
[ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down))
|
||||
@@ -620,14 +620,14 @@ STATUS_NODE(){
|
||||
########################
|
||||
## DOCKER CHECK
|
||||
########################
|
||||
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${ST_ACTION}" != "services" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
[ "${ST_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
|
||||
if [[ -v ${NODETYPE}_DOCKERS_CHECK ]] && [ "${NMG_ACTION}" != "services" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
[ "${NMG_ACTION}" == "" ] && [ "${NM_SERVICES_CHECK[${NODETYPE}]}" != "" ] && echo
|
||||
echo -e "${idsCL[Green]} Docker Service(s) Status${idsCL[Default]}"
|
||||
DIVIDER . green 52
|
||||
fi
|
||||
for docker in "${!dockers}"; do
|
||||
if [ "${ST_ACTION}" != "report" ]; then
|
||||
if [ "${NMG_ACTION}" != "report" ]; then
|
||||
c=0; cw=${PRI_CW}; spc=''
|
||||
spc1=`expr ${cw} - ${#NM_DOCKER_DESC[${docker}]}`
|
||||
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
||||
@@ -646,7 +646,7 @@ STATUS_NODE(){
|
||||
SDCHK=true
|
||||
fi
|
||||
if [ ${SDCHK} != false ]; then
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "${idsCL[White]} ${NM_DOCKER_DESC[${docker}]}$spc${idsCL[Default]}: "
|
||||
|
||||
if [ ! "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
if [ "$(${NCMD} docker ps -a | grep ${docker})" ]; then
|
||||
@@ -658,7 +658,7 @@ STATUS_NODE(){
|
||||
fi
|
||||
sleep 10s
|
||||
if [ "$(${NCMD} docker ps -q -f name=${docker})" ]; then
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[Green]}Running - Fixed"
|
||||
fi
|
||||
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "${NM_DOCKER_DESC[${docker}]} is fixed"
|
||||
@@ -666,7 +666,7 @@ STATUS_NODE(){
|
||||
echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_DOCKER_DESC[${docker}]} (docker) is fixed" >> ${NM_LOGFILE}
|
||||
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[Red]}Not Running - Could Not Fix!"
|
||||
|
||||
if [ ! -f ${NM_TMPFOLDER}/${nip}~${docker}.down ]; then
|
||||
@@ -680,7 +680,7 @@ STATUS_NODE(){
|
||||
mv ${NM_TMPFOLDER}/${nip}~${docker}.down ${NM_TMPFOLDER}/${nip}~${docker}.errtime
|
||||
fi
|
||||
toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${docker}.errtime))
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -e "${idsCL[Red]}Not running for $(SHOW_TIME ${toterrtime})${idsCL[Default]}"
|
||||
fi
|
||||
touch ${NM_TMPFOLDER}/${nip}~${docker}.down
|
||||
@@ -690,7 +690,7 @@ STATUS_NODE(){
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
echo -en "${idsCL[LightGreen]}Running"
|
||||
fi
|
||||
if [[ "${NM_SINGLESRVR_DOCKERS[${NODETYPE}]}" = *"${docker}"* ]]; then
|
||||
@@ -709,11 +709,11 @@ STATUS_NODE(){
|
||||
fi
|
||||
|
||||
else
|
||||
[ "${ST_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
|
||||
[ "${NMG_ACTION}" != "check" ] && echo -en "\033[K\r ${idsST[Bold]}${idsCL[LightRed]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}"
|
||||
rm -f ${NM_TMPFOLDER}/${nip}~*
|
||||
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
|
||||
touch ${NM_TMPFOLDER}/${nip}.down
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
echo -e "${idsCL[Red]} - Node is online, but SSH is down!${idsCL[Default]}"
|
||||
else
|
||||
@@ -738,7 +738,7 @@ STATUS_NODE(){
|
||||
[ -f ${NM_TMPFOLDER}/${nip}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)) || toterrtime=${errtime}
|
||||
fi
|
||||
|
||||
if [ "${ST_ACTION}" != "check" ]; then
|
||||
if [ "${NMG_ACTION}" != "check" ]; then
|
||||
if [ "${checkhost}" != "false" ]; then
|
||||
echo -e "${idsCL[Red]} - Node SSL has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYellow]}${LH}${idsCL[Default]}"
|
||||
else
|
||||
@@ -757,7 +757,7 @@ STATUS_NODE(){
|
||||
fi
|
||||
|
||||
fi
|
||||
[ "${ST_ACTION}" != "check" ] && echo
|
||||
[ "${NMG_ACTION}" != "check" ] && echo
|
||||
done
|
||||
|
||||
fi #GOFORCHECK
|
||||
|
||||
Reference in New Issue
Block a user