640 lines
25 KiB
Bash
Executable File
640 lines
25 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# DSMon - Disk and Basic System Monitor Script
|
|
|
|
[ "${3}" != "q" ] && source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
|
|
[ -d /opt/idssys/nodemgmt ] && source /opt/idssys/nodemgmt/defaults.inc
|
|
|
|
source /opt/idssys/dsmon/config.settings.inc
|
|
source /opt/idssys/dsmon/defaults.inc
|
|
|
|
|
|
|
|
ACTION="${1}"
|
|
|
|
RUN(){
|
|
if [ "${SERVERMON_ID}" != "" ]; then
|
|
|
|
if [ "${1}" = "hdd" ]; then
|
|
DRIVEINFO=$(df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|localhost|shm|overlay|-volume|Music|Software|//|AFS|PlexData_VG' | awk '{ print $1 " " $2 " " $4 }')
|
|
DRIVEINFO=(${DRIVEINFO})
|
|
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
|
|
|
DRIVE_INFO=()
|
|
|
|
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
|
ii=$((${i}*3))
|
|
ia=${DRIVEINFO[${ii}]}
|
|
ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024")
|
|
ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024")
|
|
ia=${ia//\/dev\/mapper\//}
|
|
ia=${ia//\/dev\//}
|
|
|
|
if [[ "${ia}" = *"sda"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [[ "${ia}" = *"nvme"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [[ "${ia}" = *"mmcblk"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
else
|
|
# echo "${ia};${ib};${ic}"
|
|
DRIVE_INFO+=("${ia};${ib};${ic}")
|
|
fi
|
|
|
|
done
|
|
|
|
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');"
|
|
# echo $QRY
|
|
MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "${QRY}"
|
|
|
|
elif [ "${1}" = "sys" ]; then
|
|
|
|
MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }')
|
|
CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}')
|
|
|
|
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU// /};${MEMORY// /}');"
|
|
|
|
MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "${QRY}"
|
|
fi
|
|
|
|
SERVERHOSTNAME=(`hostname`)
|
|
SERVERIP=(`hostname -I`)
|
|
SERVERIPS=''
|
|
for nip in "${SERVERIP[@]}"; do
|
|
if [[ "${nip}" == *"."* ]] && [[ "${nip}" != *"172"* ]] && [[ "${nip}" != *"169"* ]]; then
|
|
if [ "${SERVERIPS}" = "" ]; then
|
|
SERVERIPS=${nip}
|
|
else
|
|
SERVERIPS=${SERVERIPS}";"${nip}
|
|
fi
|
|
fi
|
|
done
|
|
|
|
thresholdlog=()
|
|
for KEY in "${!THRESHOLD[@]}"; do
|
|
thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}")
|
|
done
|
|
|
|
QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';"
|
|
MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "${QRY}"
|
|
|
|
|
|
fi
|
|
}
|
|
|
|
|
|
CHECK(){
|
|
rm -f /tmp/dscheck.*
|
|
start=`date +%s`
|
|
|
|
C_CW=40
|
|
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
declare -A host_limits
|
|
declare -A host_ids
|
|
declare -a hostnames_sort
|
|
|
|
if [ "${1}" == "report" ]; then
|
|
unset idsCL idsBG idsST
|
|
idsCL=('')
|
|
idsBG=('')
|
|
idsST=('')
|
|
fi
|
|
|
|
if [ "${1}" != "report" ]; then
|
|
MSG="Determining hosts and preparing"
|
|
c=0; cw=${C_CW}; spc=''; spc1=$(expr ${cw} - ${#MSG}); until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
|
|
fi
|
|
if [ -n "${1}" ] && [ "${1}" != "report" ]; then
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
hostip=$(echo $hostip | cut -d ";" -f1)
|
|
# echo "$hostid - $hostip - $hostname - $hostlimits"
|
|
host_ip[${hostid}]=$hostip
|
|
host_name[${hostid}]=$hostname
|
|
host_limits[${hostid}]=$hostlimits
|
|
host_ids[$hostname]=$hostid
|
|
hostnames_sort+=("$hostname")
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 AND (host LIKE '%${1}%' OR ip LIKE '%${1}%')")
|
|
else
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
hostip=$(echo $hostip | cut -d ";" -f1)
|
|
# echo "$hostid - $hostip - $hostname - $hostlimits"
|
|
host_ip[${hostid}]=$hostip
|
|
host_name[${hostid}]=$hostname
|
|
host_limits[${hostid}]=$hostlimits
|
|
host_ids[$hostname]=$hostid
|
|
hostnames_sort+=("$hostname")
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
|
|
fi
|
|
|
|
IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS
|
|
[ "${1}" != "report" ] && echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}"
|
|
|
|
if [ "${1}" != "report" ]; then
|
|
MSG="Starting jobs to collect host info"
|
|
c=0; cw=${C_CW}; spc=''; spc1=$(expr ${cw} - ${#MSG}); until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en "${idsCL[LightCyan]}${MSG}${spc}: "
|
|
fi
|
|
started_hosts=0
|
|
for hostname in "${hostnames_sort[@]}"; do
|
|
while [ $(jobs -rp | wc -l) -ge 15 ]; do sleep 0.5; done
|
|
unbuffer bash -c "/usr/local/bin/dsmon runcheck '${hostname}'" > /tmp/dscheck.${hostname}.results &
|
|
((started_hosts++))
|
|
[ "${1}" != "report" ] && printf "\r${idsCL[LightCyan]}${MSG}${spc}: Started on %d of %d hosts %s${idsCL[Default]}" "${started_hosts}" "${#hostnames_sort[@]}" $([[ "$started_hosts" -lt "${#hostnames_sort[@]}" ]] && echo " ..." || echo "")
|
|
|
|
done
|
|
[ "${1}" != "report" ] && echo -e "\r\033[2K${idsCL[LightCyan]}${MSG}${spc}: ${idsCL[LightGreen]}Done${idsCL[Default]}"
|
|
echo
|
|
shopt -s nullglob
|
|
nc_count=0; lc_count=0; completed=false; scanstart=$(date +%s)
|
|
until [ "${completed}" == "true" ]; do
|
|
until [ $(ls /tmp/dscheck.*.done 2>/dev/null | wc -l) -gt 0 ]; do tmp=tmp; done
|
|
[ ${nc_count} -gt 0 ] && echo -en "\r\033[2K\033[1A\r"
|
|
for host in /tmp/dscheck.*.done; do
|
|
NTS=$(grep -oP '(?<=dscheck\.).*?(?=\.done)' <<< "${host}")
|
|
if [ -n "$NTS" ]; then
|
|
[ ${nc_count} -eq 0 ] && echo -en "\r\033[2K\033[1A\r"
|
|
[ "${1}" != "report" ] && printf "%s\n" "$(cat /tmp/dscheck.${NTS}.results)"
|
|
rm -f /tmp/dscheck.${NTS}.* >/dev/null 2>&1
|
|
((nc_count++))
|
|
fi
|
|
done
|
|
|
|
if [ "${1}" != "report" ]; then
|
|
[ ${nc_count} -gt ${lc_count} ] && echo
|
|
printf "\r${idsCL[LightCyan]}Completed on %d of %d hosts %s${idsCL[Default]}" "${nc_count}" "${#hostnames_sort[@]}" $([[ "$nc_count" -lt "${#hostnames_sort[@]}" ]] && echo " ..." || echo "")
|
|
fi
|
|
|
|
if [ $(($(date +%s)-scanstart)) -gt 300 ]; then
|
|
SENDNOTICE "DSmon Scan Stuck" "DSmon Check scan has been running for 10mins, klling all bash...." 1
|
|
pw service all stop
|
|
killall bash
|
|
exit 1
|
|
fi
|
|
if [ ${nc_count} -ge ${#hostnames_sort[@]} ]; then
|
|
completed=true
|
|
break
|
|
fi
|
|
lc_count=${nc_count}
|
|
sleep 1s
|
|
done
|
|
|
|
end=`date +%s`; runtime=$((end-start)); echo -e "\nScript Runtime: ${runtime}s"
|
|
echo
|
|
}
|
|
|
|
RUNCHECK(){
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
declare -A host_limits
|
|
declare -A host_ids
|
|
declare -a hostnames_sort
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
hostip=$(echo $hostip | cut -d ";" -f1)
|
|
# echo "$hostid - $hostip - $hostname - $hostlimits"
|
|
host_ip[${hostid}]=$hostip
|
|
host_name[${hostid}]=$hostname
|
|
host_limits[${hostid}]=$hostlimits
|
|
host_ids[$hostname]=$hostid
|
|
hostnames_sort+=("$hostname")
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 AND host='${1}'")
|
|
RUN_CHECK ${1}
|
|
}
|
|
|
|
RUN_CHECK(){
|
|
hostname=${1}
|
|
touch /tmp/dscheck.${hostname}.running
|
|
hostid=${host_ids[$hostname]}
|
|
|
|
MSG=" ${host_name[$hostid]}"
|
|
MSG2="${host_ip[$hostid]} "
|
|
c=0; cw=65; spc=''; spc1=`expr ${cw} - ${#MSG}`; until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
c=0; cw2=20; spcB=''; spc1=`expr ${cw2} - ${#MSG2}`; until [ $c = ${spc1} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
|
|
DIVIDER 1 . ${cw}
|
|
echo -e "${idsBG[Blue]}${idsCL[White]}${MSG}${spc}${MSG2}${spcB}${idsST[Reset]}"
|
|
DIVIDER . . ${cw}
|
|
|
|
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
declare -A host_limits_tmp
|
|
hostlimits=(${host_limits[${hostid}]})
|
|
for hl in ${hostlimits[@]}; do
|
|
hlname=$(echo $hl | cut -d ":" -f1)
|
|
hllim=$(echo $hl | cut -d ":" -f2)
|
|
host_limits_tmp[$hlname]=$hllim
|
|
done
|
|
|
|
DRIVEINFO=$(${SSHCMD} root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|localhost|shm|overlay|-volume|Music|Software|//|AFS|PlexData_VG' | awk '{ print $1 " " $2 " " $4 }')
|
|
DRIVEINFO=(${DRIVEINFO})
|
|
|
|
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
|
declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME
|
|
declare -a DRIVES
|
|
|
|
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
|
ii=$((${i}*3))
|
|
dname=${DRIVEINFO[${ii}]}
|
|
dname=${dname//\/dev\/mapper\//}
|
|
dname=${dname//\/dev\//}
|
|
dname=${dname//\/dev\//}
|
|
dname_short=${dname#*vg-}
|
|
dname_short=${dname_short%*_v}
|
|
|
|
[ "$dname_short" = "" ] && dname_short=$dname
|
|
|
|
if [[ "${ia}" = *"sda"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [[ "${ia}" = *"nvme"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [[ "${ia}" = *"mmcblk"* ]] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
else
|
|
dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10")
|
|
dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10")
|
|
dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100))
|
|
|
|
DRIVEINFO_TOT[$dname]=$dtot
|
|
DRIVEINFO_FREE[$dname]=$dfree
|
|
DRIVEINFO_FREEPER[$dname]=$dfreeper
|
|
DRIVEINFO_SHORTNAME[$dname]=$dname_short
|
|
DRIVES+=("$dname")
|
|
fi
|
|
done
|
|
|
|
IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS
|
|
|
|
c=0; cw=18; spcA=''; spc1=`expr ${cw} - 7`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcB=''; spc2=`expr ${cw} - 4`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcC=''; spc3=`expr ${cw} - 5`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
|
|
c=0; cw=14; spcD=''; spc4=`expr ${cw} - 3`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
|
|
echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA} FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN % / CRIT %${idsCL[Default]}"
|
|
|
|
for DRIVE in "${DRIVES[@]}"; do
|
|
|
|
[ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81
|
|
|
|
WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]}))
|
|
CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2))
|
|
|
|
c=0; cw=18; spcA=''; spc1=`expr ${cw} - ${#DRIVEINFO_SHORTNAME[$DRIVE]}`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcB=''; spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcC=''; spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
|
|
c=0; cw=14; spcD=''; spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
|
|
|
|
if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then
|
|
fs_status='CRITICAL'
|
|
fs_priority=2
|
|
fs_status_color='RedBold'
|
|
|
|
elif (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${WARNING_LEVEL}") )); then
|
|
fs_status='Warning'
|
|
fs_priority=1
|
|
fs_status_color='Yellow'
|
|
|
|
elif (( $(bc <<<"${DRIVEINFO_FREE[$DRIVE]} < 5") )); then
|
|
fs_status='Warning'
|
|
fs_priority=1
|
|
fs_status_color='Magenta'
|
|
else
|
|
fs_status='Normal'
|
|
fs_priority=0
|
|
fs_status_color='Green'
|
|
fi
|
|
|
|
D_WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${WARNING_LEVEL}))
|
|
D_CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n 100-${CRITICAL_LEVEL}))
|
|
echo -e "${idsCL[Cyan]} ${DRIVEINFO_SHORTNAME[$DRIVE]}${spcA}${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${idsST[Reset]}${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${D_WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[RedBold]}${D_CRITICAL_LEVEL}%${idsST[Reset]}${idsCL[Cyan]} )${idsCL[Default]}"
|
|
|
|
if [ "${fs_status}" != "Normal" ] && [ "${1}" == "report" ]; then
|
|
# echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)\n\n$(date)" | mail -s "Free Space ${fs_status}: '${host_name[$hostid]}'" ${ALERT_EMAIL}
|
|
SENDNOTICE "Free Space ${fs_status}: '${host_name[$hostid]}'" "${host_name[$hostid]} - (${host_ip[$hostid]})
|
|
|
|
${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]})percent" ${fs_priority}
|
|
fi
|
|
|
|
done
|
|
|
|
unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME DRIVES DRIVEINFO host_limits_tmp
|
|
|
|
maillogsize=$(${SSHCMD} root@${host_ip[$hostid]} du -hs /var/log/mail.log | cut -d'/' -f 1)
|
|
if [[ "${maillogsize//[[:blank:]]/}" = *'G'* ]]; then
|
|
if [ ! -f /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent ] || ([ -f /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent ] && [ $(expr $(date +%s) - $(stat -c %Y /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent)) -gt 3600 ]); then
|
|
SENDNOTICE "Mail Log Getting Big" "Mail log getting big (${maillogsize//[[:blank:]]/}) on ${hostname} [${host_ip[$hostid]}]" 1
|
|
touch /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent
|
|
fi
|
|
elif [ -f /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent ]; then
|
|
rm -f /tmp/dsmon.mail-log.${host_ip[$hostid]}.errorsent
|
|
fi
|
|
|
|
else
|
|
echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}"; echo
|
|
fi
|
|
mv /tmp/dscheck.${hostname}.running /tmp/dscheck.${hostname}.done
|
|
}
|
|
|
|
SETUPSSH(){
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
|
|
host_name[${hostid}]=$hostname
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
|
|
|
|
for hostid in "${!host_ip[@]}"; do
|
|
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
echo -en "\n${idsCL[LightCyan]}${host_name[$hostid]} - Checking for Public SSH Key ... ${idsCL[Default]}"
|
|
if [ "$(ssh -o BatchMode=yes -o ConnectTimeout=3 root@${nip} echo ok 2>&1)" == "ok" ]; then
|
|
echo -e "${idsCL[Green]}Already Installed${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Yellow]}Copying to Host ...${idsCL[Default]}"
|
|
ssh-copy-id -f root@${host_ip[$hostid]}
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
|
|
RUNCMD(){
|
|
start=`date +%s`
|
|
rm -f /tmp/.dsmon.runcmd.*
|
|
echo -e "\n${idsCL[White]}The following command will be run on all nodes:"
|
|
echo -e "${idsST[Bold]}'${idsCL[LightCyan]}$(echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}" | awk '{$1=$1};1')${idsCL[White]}'${idsCL[Default]}${idsST[Reset]}\n"
|
|
ncount=0
|
|
|
|
# while read hostid hostname hostip hostlimits; do
|
|
# if [ "${hostname}" != "host" ]; then
|
|
# ${SSHCMD} root@${hostip} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22}
|
|
# fi
|
|
# done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
|
|
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "${hostname}" != "host" ]; then
|
|
RUNCMD_SUB $(echo ${hostip} | cut -d ";" -f1) ${hostname} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} &
|
|
((ncount++))
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
|
|
|
|
nc_count=0; lcount=0; completed=false
|
|
until [ "${completed}" == "true" ]; do
|
|
for nodecmd in /tmp/.dsmon.runcmd.*.done; do
|
|
# HOST=$(grep -oP '(?<=runcmd.).*?(?=.done)' <<< "${nodecmd}")
|
|
# if [ "${HOST}" != "*" ]; then
|
|
# if [[ "${nodecmd}" != *"*"* ]]; then
|
|
# nodecmd_output=$(cat ${nodecmd})
|
|
# [ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n"
|
|
# fi
|
|
# rm -f ${nodecmd}
|
|
|
|
if [[ "${nodecmd}" != *"*"* ]]; then
|
|
nodecmd_output=$(cat ${nodecmd})
|
|
[ "${nodecmd_output}" != "" ] && echo -e "${nodecmd_output}\n"
|
|
rm -f ${nodecmd}
|
|
((nc_count++))
|
|
fi
|
|
done
|
|
if [ ${nc_count} -eq ${ncount} ]; then
|
|
echo -e "${idsCL[LightGreen]}${nc_count} of ${ncount} completed, the command was sent to all nodes\n"
|
|
completed=true
|
|
break
|
|
fi
|
|
[ ${lcount} -ne ${nc_count} ] && echo -e "${idsCL[LightYellow]}${nc_count} of ${ncount} completed ...\n"
|
|
lcount=${nc_count}
|
|
|
|
sleep 2s
|
|
done
|
|
|
|
echo
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "Script Runtime: ${runtime}"
|
|
echo
|
|
}
|
|
RUNCMD_SUB(){
|
|
ip=${1}
|
|
host=${2}
|
|
tmpfile=/tmp/.dsmon.runcmd.${host// /-}
|
|
touch ${tmpfile}.running > /dev/null 2>&1
|
|
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host}${idsCL[Default]}${idsST[Reset]}" >> ${tmpfile}.running
|
|
if ssh_output=$(${SSHCMD} root@${ip} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} 2>&1) ; then
|
|
echo -e "${idsCL[LightGreen]}SUCCESS: ${ssh_output}${idsCL[Default]}" >> ${tmpfile}.running
|
|
else
|
|
if [ "${ssh_output}" == "" ]; then
|
|
echo -e "${idsCL[Red]} ${host} [${ip}] - Timeout${idsCL[Default]}" >> ${tmpfile}.running
|
|
else
|
|
echo -e "${idsCL[Yellow]}${ssh_output}${idsCL[Default]}" >> ${tmpfile}.running
|
|
fi
|
|
fi
|
|
mv ${tmpfile}.running ${tmpfile}.done > /dev/null 2>&1
|
|
}
|
|
RUNCMDSLOW(){
|
|
start=`date +%s`
|
|
echo -e "\n${idsCL[White]}The following command will be run on all nodes:"
|
|
echo -e "${idsST[Bold]}'${idsCL[LightCyan]}$(echo "${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20}" | awk '{$1=$1};1')${idsCL[White]}'${idsCL[Default]}${idsST[Reset]}\n"
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "${hostname}" != "host" ]; then
|
|
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${hostname} - $(echo ${hostip} | cut -d ";" -f1)${idsCL[Default]}${idsST[Reset]}"
|
|
${SSHCMD} root@$(echo ${hostip} | cut -d ";" -f1) ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21} ${22} </dev/null
|
|
echo
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
|
|
|
|
echo -e "${idsCL[LightGreen]}Complete, the command was ran on all nodes\n"
|
|
|
|
echo
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "Script Runtime: ${runtime}"
|
|
echo
|
|
}
|
|
|
|
UPDATENODES(){
|
|
start=`date +%s`
|
|
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "${hostname}" != "host" ]; then
|
|
echo -e "${idsCL[LightCyan]}Updating DSMON on '${hostname}'${idsCL[Default]}"
|
|
${SSHCMD} root@$(echo $hostip | cut -d ";" -f1) "dsmon update" </dev/null
|
|
echo
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
|
|
|
|
echo
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "Script Runtime: ${runtime}"
|
|
echo
|
|
|
|
}
|
|
|
|
GETTSIP_NODE(){
|
|
TSI=$(/sbin/ip link | grep tailscale0)
|
|
if [ ${#TSI} != 0 ]; then
|
|
TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1)
|
|
hostname=$(hostname -f); hostname=${hostname,,}
|
|
([ "${hostname}" == "" ] || [[ "${hostname}" != *".scity"* ]]) && hostname=$(hostname) && hostname="${hostname,,}.ts.scity.us" || hostname=${hostname/.scity/.ts.scity}
|
|
echo "Tailscale running, TSIP: ${TSIP}"
|
|
if [ "${NM_PDNS_USER}" != "" ]; then
|
|
echo -en "Updating DNS for '${hostname}' ... "
|
|
dyndns=$(/usr/bin/curl -s "https://${NM_PDNS_USER}:${NM_PDNS_PASS}@ldns.scity.us/nic/update?hostname=${hostname}&myip=${TSIP}")
|
|
echo "Done (${dyndns})"
|
|
fi
|
|
else
|
|
echo "Not running Tailscale"
|
|
fi
|
|
}
|
|
|
|
GETCRONTABS(){
|
|
start=`date +%s`
|
|
rtpsswd=${1}
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
|
|
[ ! -d ${DS_CRONTAB_FOLDER} ] && mkdir ${DS_CRONTAB_FOLDER} && chmod -Rf 0777 ${DS_CRONTAB_FOLDER}
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
|
|
host_name[${hostid}]=$hostname
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
|
|
|
|
if [ "${rtpsswd}" == "" ]; then
|
|
read -p "Enter the password for the 'nm_crontab_user' account on MySQL-Manager: " rtpsswd
|
|
echo
|
|
fi
|
|
|
|
for hostid in "${!host_ip[@]}"; do
|
|
# echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - ${host_ip[$hostid]}${idsCL[Default]}${idsST[Reset]}"
|
|
# checkhost=$(CHECK_HOST ${host_ip[$hostid]})
|
|
# if [ "${checkhost}" != "false" ]; then
|
|
# echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Gathering Crontabs ... ${idsCL[Default]}${idsST[Reset]}"
|
|
${SSHCMD} root@${host_ip[$hostid]} dsmon get-crontab ${rtpsswd}
|
|
# else
|
|
# echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
|
|
# fi
|
|
done
|
|
|
|
echo
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "Script Runtime: ${runtime}"
|
|
echo
|
|
|
|
}
|
|
GETCRONTAB(){
|
|
if [ "${1}" != "" ]; then
|
|
if ! command -v sshpass > /dev/null; then
|
|
apt install -y sshpass > /dev/null 2>&1
|
|
fi
|
|
if [ "${nmip}" != "" ]; then
|
|
while read host hostname; do
|
|
if [ "${host}" != "host" ] && [ "${host}" != "" ]; then
|
|
# echo -e "${SERVERMON_ID}) ${host} : ${hostname}\nmysqlip=${mysqlip}\n"
|
|
crontab -l | sshpass -p${1} ${SSHCMD} -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${host}.crontab"
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" ${mysqlcmd} ${mysql_conn} -u sysmoninsert -e "SELECT host,hostname FROM servermonitor.hosts WHERE id=${SERVERMON_ID}")
|
|
fi
|
|
|
|
# [ "${nmip}" != "" ] && crontab -l | sshpass -p${1} ssh -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${hostname}.crontab"
|
|
|
|
else
|
|
echo "No user password specified"
|
|
fi
|
|
}
|
|
FIXCRONTAB(){
|
|
crontab -l | grep -v 'dsmon hdd' | crontab -
|
|
crontab -l | grep -v 'dsmon sys' | crontab -
|
|
(crontab -l ; echo "$(($RANDOM % 59)) */1 * * * /usr/local/bin/dsmon hdd >/dev/null 2>&1")| crontab -
|
|
(crontab -l ; echo "*/5+$(($RANDOM % 4)) * * * * /usr/local/bin/dsmon sys >/dev/null 2>&1")| crontab -
|
|
crontab -l | grep "dsmon"
|
|
}
|
|
|
|
# case ${ACTION} in
|
|
# check) CHECK ${2};;
|
|
# setupssh) SETUPSSH;;
|
|
# get-crontabs) GETCRONTABS ${2};;
|
|
# get-crontab) GETCRONTAB ${2};;
|
|
# fix-crontab) FIXCRONTAB;;
|
|
# gettsip) GETTSIP;;
|
|
# gettsip-node) GETTSIP_NODE;;
|
|
# run) RUN ${2};;
|
|
# runcmd) RUNCMD ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11};;
|
|
# update);;
|
|
# *) RUN ${ACTION};;
|
|
# esac
|
|
|
|
if [[ "${noheader}" != *" ${1} "* ]] && [[ "${noheader}" != *" ${2} "* ]]; then
|
|
echo
|
|
DIVIDER . lightGreen 85
|
|
msg="| Disk Space Monitor (DSMon) "
|
|
msg2="(ver-${VERS})"
|
|
c=0; spc=''; spc1=`expr 62 - ${#msg}`; until [ ${c} = ${spc1} ]; do spc="${spc} "; c=`expr ${c} + 1`; done
|
|
c=0; spcB=''; spc1=`expr 22 - ${#msg2}`; until [ ${c} = ${spc1} ]; do spcB="${spcB} "; c=`expr ${c} + 1`; done
|
|
echo -e "${idsCL[LightGreen]}${msg}${idsCL[Default]}${spc}${idsCL[Yellow]}${msg2}${spcB}${idsCL[LightGreen]}|${idsCL[Default]}"
|
|
DIVIDER . lightGreen 85
|
|
fi
|
|
case ${ACTION} in
|
|
check) CHECK ${2};;
|
|
runcheck) RUNCHECK ${2};;
|
|
setupssh) SETUPSSH;;
|
|
get-crontabs) GETCRONTABS ${2};;
|
|
get-crontab) GETCRONTAB ${2};;
|
|
fix-crontab) FIXCRONTAB;;
|
|
gettsip) GETTSIP;;
|
|
gettsip-node) GETTSIP_NODE;;
|
|
run) RUN ${2};;
|
|
runcmd) RUNCMD ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21};;
|
|
runcmdslow) RUNCMDSLOW ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11} ${12} ${13} ${14} ${15} ${16} ${17} ${18} ${19} ${20} ${21};;
|
|
update);;
|
|
update-nodes) UPDATENODES;;
|
|
*)
|
|
if [ "${ACTION}" == "hdd" ] || [ "${ACTION}" == "sys" ]; then
|
|
RUN ${ACTION}
|
|
else
|
|
echo -e "${idsCL[White]} Usage: ${idsCL[LightCyan]}dsmon {option}"
|
|
echo
|
|
echo -e " ${idsCL[Yellow]}check ${idsCL[Default]}--> Run a Disk Space check for all hosts"
|
|
echo -e " ${idsCL[Yellow]}setupssh ${idsCL[Default]}--> Configure direct SSH access to all hosts"
|
|
echo -e " ${idsCL[Yellow]}get-crontabs ${idsCL[Default]}--> Get root crontabs from all hosts"
|
|
echo -e " ${idsCL[Yellow]}gettsip ${idsCL[Default]}--> Check Sensors"
|
|
echo -e " ${idsCL[Yellow]}gettsip-node ${idsCL[Default]}--> Check Sensors"
|
|
echo -e " ${idsCL[Yellow]}run {type} ${idsCL[Default]}--> Monitor Check to run"
|
|
echo -e " ${idsCL[LightYellow]} type = { hdd / sys } ${idsCL[Default]}--> Disk Space or System Resources"
|
|
echo -e " ${idsCL[Yellow]}runcmd ${idsCL[Default]}--> Run command on all hosts, in parallel"
|
|
echo -e " ${idsCL[Yellow]}runcmd-slow ${idsCL[Default]}--> Run command on all hosts, one at a time"
|
|
echo -e " ${idsCL[Yellow]}update ${idsCL[Default]}--> Update DSMon"
|
|
echo
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
exit 0
|
|
|
|
|