Files
dsmon/dsmon.sh
2023-10-09 17:48:38 -05:00

439 lines
16 KiB
Bash
Executable File

#!/usr/bin/env bash
# DSMon - Disk and Basic System Monitor Script
source /opt/idssys/defaults/colors.inc
source /opt/idssys/dsmon/config.settings.inc
source /opt/idssys/dsmon/defaults.inc
source /opt/idssys/defaults/default.inc
action="$1"
RUN(){
if [ "${SERVERMON_ID}" != "" ]; then
if [ "${1}" = "hdd" ]; then
DRIVEINFO=$(df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
DRIVE_INFO=()
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
ii=$((${i}*3))
ia=${DRIVEINFO[${ii}]}
ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024")
ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024")
ia=${ia//\/dev\/mapper\//}
ia=${ia//\/dev\//}
if [ "${ia}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${ia}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${ia}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${ia}" = "nvme1n1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 4096 ]; then
systempartition=true
elif [ "${ia}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
else
#echo "${ia};${ib};${ic}"
DRIVE_INFO+=("${ia};${ib};${ic}")
fi
done
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');"
# echo $QRY
MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "${QRY}"
elif [ "${1}" = "sys" ]; then
MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }')
CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}')
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU// /};${MEMORY}');"
MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "${QRY}"
fi
SERVERHOSTNAME=(`hostname`)
SERVERIP=(`hostname -I`)
SERVERIPS=''
for nip in "${SERVERIP[@]}"; do
if [[ "${nip}" == *"."* ]] && [[ "${nip}" != *"172"* ]] && [[ "${nip}" != *"169"* ]]; then
if [ "${SERVERIPS}" = "" ]; then
SERVERIPS=${nip}
else
SERVERIPS=${SERVERIPS}";"${nip}
fi
fi
done
thresholdlog=()
for KEY in "${!THRESHOLD[@]}"; do
thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}")
done
QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';"
MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "${QRY}"
fi
}
CHECK(){
start=`date +%s`
declare -A host_ip
declare -A host_name
declare -A host_limits
declare -A host_ids
declare -a hostnames_sort
if [ "${1}" == "report" ]; then
unset idsCL idsBG idsST
idsCL=('')
idsBG=('')
idsST=('')
fi
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
hostip=$(echo $hostip | cut -d ";" -f1)
# echo "$hostid - $hostip - $hostname - $hostlimits"
host_ip[${hostid}]=$hostip
host_name[${hostid}]=$hostname
host_limits[${hostid}]=$hostlimits
host_ids[$hostname]=$hostid
hostnames_sort+=("$hostname")
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS
#for hostid in "${!host_ip[@]}"; do
for hostname in "${hostnames_sort[@]}"; do
hostid=${host_ids[$hostname]}
if [ "${1}" != "report" ]; then
echo -e "${idsST[Bold]}"; DIVIDER false lightCyan 85
echo -e "${idsCL[LightCyan]} ${host_name[$hostid]} - Drive Space Check${idsCL[Default]}"
DIVIDER false lightCyan 85
fi
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
declare -A host_limits_tmp
hostlimits=(${host_limits[${hostid}]})
for hl in ${hostlimits[@]}; do
hlname=$(echo $hl | cut -d ":" -f1)
hllim=$(echo $hl | cut -d ":" -f2)
host_limits_tmp[$hlname]=$hllim
done
DRIVEINFO=$(ssh root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }')
DRIVEINFO=(${DRIVEINFO})
# for x in "${!DRIVEINFO[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO[$x]}" ; done
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME
declare -a DRIVES
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
ii=$((${i}*3))
dname=${DRIVEINFO[${ii}]}
dname=${dname//\/dev\/mapper\//}
dname=${dname//\/dev\//}
dname_short=${dname#*vg-}
dname_short=${dname_short%*_v}
[ "$dname_short" = "" ] && dname_short=$dname
if [ "${dname}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${dname}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${dname}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${dname}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
elif [ "${dname}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
systempartition=true
else
dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10")
dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10")
dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100))
DRIVEINFO_TOT[$dname]=$dtot
DRIVEINFO_FREE[$dname]=$dfree
DRIVEINFO_FREEPER[$dname]=$dfreeper
DRIVEINFO_SHORTNAME[$dname]=$dname_short
DRIVES+=("$dname")
fi
done
IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS
# for x in "${!DRIVEINFO_TOT[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_TOT[$x]}" ; done
# echo
# for x in "${!DRIVEINFO_FREE[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREE[$x]}" ; done
# echo
# for x in "${!DRIVEINFO_FREEPER[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREEPER[$x]}" ; done
# echo
if [ "${1}" != "report" ]; then
c=0; cw=18; spcA=''; spc1=`expr ${cw} - 7`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
c=0; cw=10; spcB=''; spc2=`expr ${cw} - 4`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
c=0; cw=10; spcC=''; spc3=`expr ${cw} - 5`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
c=0; cw=14; spcD=''; spc4=`expr ${cw} - 3`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA} FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN % / CRIT %${idsCL[Default]}"
# pf="%-20s %10s %10s %12s %50s\n"
# printf "${pf}" "${idsCL[White]}DRIVE" "FREE GB" "FREE %" "TOTAL GB" "WARN % / CRIT %${idsCL[Default]}"
fi
for DRIVE in "${DRIVES[@]}"; do
[ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81
WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]}))
CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2))
if [ "${1}" != "report" ]; then
c=0; cw=18; spcA=''; spc1=`expr ${cw} - ${#DRIVEINFO_SHORTNAME[$DRIVE]}`; until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
c=0; cw=10; spcB=''; spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}`; until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
c=0; cw=10; spcC=''; spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}`; until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
c=0; cw=14; spcD=''; spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}`; until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
fi
if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then
fs_status='CRITICAL'
fs_priority=2
fs_status_color='RedBold'
elif (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${WARNING_LEVEL}") )); then
fs_status='Warning'
fs_priority=1
fs_status_color='Yellow'
elif (( $(bc <<<"${DRIVEINFO_FREE[$DRIVE]} < 5") )); then
fs_status='Warning'
fs_priority=1
fs_status_color='Magenta'
else
fs_status='Normal'
fs_priority=0
fs_status_color='Green'
fi
D_WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${WARNING_LEVEL}))
D_CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n 100-${CRITICAL_LEVEL}))
[ "${1}" != "report" ] && echo -e "${idsCL[Cyan]} ${DRIVEINFO_SHORTNAME[$DRIVE]}${spcA}${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${idsST[Reset]}${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${D_WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[RedBold]}${D_CRITICAL_LEVEL}%${idsST[Reset]}${idsCL[Cyan]} )${idsCL[Default]}"
# [ "${1}" != "report" ] && printf "${pf}" "${idsCL[Cyan]}${DRIVEINFO_SHORTNAME[$DRIVE]}" "${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB" "${DRIVEINFO_FREEPER[$DRIVE]}%" "${DRIVEINFO_TOT[$DRIVE]} GB" "${idsCL[Cyan]}(${idsCL[Default]}${D_WARNING_LEVEL}%${idsCL[Cyan]}/${idsCL[Default]}${D_CRITICAL_LEVEL}%${idsCL[Cyan]})(${idsCL[Yellow]}${D_WARNING_LEVEL}%${idsCL[Cyan]}/${idsCL[Red]}${D_CRITICAL_LEVEL}%${idsCL[Cyan]})${idsCL[Default]}"
if [ "${fs_status}" != "Normal" ] && [ "${1}" == "report" ]; then
# echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)\n\n$(date)" | mail -s "Free Space ${fs_status}: '${host_name[$hostid]}'" ${ALERT_EMAIL}
SENDNOTICE "Free Space ${fs_status}: '${host_name[$hostid]}'" "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)" ${fs_priority}
fi
SENDNOTICE "Free Space ${fs_status}: '${host_name[$hostid]}'" "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)" ${fs_priority}
done
unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME DRIVES DRIVEINFO host_limits_tmp
else
[ "${1}" != "report" ] && echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}"; echo
fi
done
end=`date +%s`
runtime=$((end-start))
echo "Script Runtime: ${runtime}"
}
SETUPSSH(){
declare -A host_ip
declare -A host_name
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
host_name[${hostid}]=$hostname
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
for hostid in "${!host_ip[@]}"; do
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - SSH KEY COPY${idsCL[Default]}${idsST[Reset]}"
ssh-copy-id root@${host_ip[$hostid]}
fi
done
}
RUNCMD(){
start=`date +%s`
declare -A host_ip
declare -A host_name
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
host_name[${hostid}]=$hostname
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
for hostid in "${!host_ip[@]}"; do
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running command: ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10}${idsCL[Default]}${idsST[Reset]}"
ssh -tq root@${host_ip[$hostid]} ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10}
else
echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
fi
done
echo
end=`date +%s`
runtime=$((end-start))
echo "Script Runtime: ${runtime}"
echo
}
GETTSIP(){
start=`date +%s`
declare -A host_ip
declare -A host_name
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
host_name[${hostid}]=$hostname
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 AND ip LIKE '%100.100.%' ORDER BY host ASC")
for hostid in "${!host_ip[@]}"; do
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
ssh -tq root@${host_ip[$hostid]} "dsmon update && dsmon gettsip-node"
else
echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
fi
done
echo
end=`date +%s`
runtime=$((end-start))
echo "Script Runtime: ${runtime}"
echo
}
GETTSIP_NODE(){
TSI=$(/sbin/ip link | grep tailscale0)
if [ ${#TSI} != 0 ]; then
TSIP=$(/sbin/ip -o -4 addr list tailscale0 | awk '{print $4}' | cut -d/ -f1)
hostname=$(hostname -f); hostname=${hostname,,}
([ "${hostname}" == "" ] || [[ "${hostname}" != *".scity"* ]]) && hostname=$(hostname) && hostname="${hostname,,}.ts.scity.us" || hostname=${hostname/.scity/.ts.scity}
echo "Tailscale running, TSIP: ${TSIP}"
echo -en "Updating DNS for '${hostname}' ... "
dyndns=$(/usr/bin/curl -s "https://systems:Dcs9613@dns.scity.us/nic/update?hostname=${hostname}&myip=${TSIP}")
echo "Done (${dyndns})"
else
echo "Not running Tailscale"
fi
}
GETCRONTABS(){
start=`date +%s`
declare -A host_ip
declare -A host_name
[ ! -d ${DS_CRONTAB_FOLDER} ] && mkdir ${DS_CRONTAB_FOLDER} && chmod -Rf 0777 ${DS_CRONTAB_FOLDER}
while read hostid hostname hostip hostlimits; do
if [ "$hostid" != "id" ]; then
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
host_name[${hostid}]=$hostname
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0 ORDER BY host ASC")
if [ "${1}" == "" ]; then
read -p "Enter the password for the 'nm_crontab_user' account on MySQL-Manager: " rtpsswd
echo
else
rtpsswd=${1}
fi
for hostid in "${!host_ip[@]}"; do
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
if [ "${checkhost}" != "false" ]; then
echo -en "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Gathering Crontabs ... ${idsCL[Default]}${idsST[Reset]}"
ssh -tq root@${host_ip[$hostid]} dsmon get-crontab ${rtpsswd}
else
echo -e "${idsST[Bold]}${idsCL[Red]} ${host_name[$hostid]} - Timeout${idsCL[Default]}${idsST[Reset]}"
fi
done
echo
end=`date +%s`
runtime=$((end-start))
echo "Script Runtime: ${runtime}"
echo
}
GETCRONTAB(){
if [ "${1}" != "" ]; then
if ! command -v sshpass > /dev/null; then
apt install -y sshpass > /dev/null 2>&1
fi
if [ "${nmip}" != "" ]; then
while read host hostname; do
if [ "${host}" != "host" ]; then
crontab -l | sshpass -p${1} ssh -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${host}.crontab"
echo -e "${idsCL[Green]} done${idsCL[Default]}"
fi
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h ${mysqlip} -u sysmoninsert -e "SELECT host,hostname FROM servermonitor.hosts WHERE id=${SERVERMON_ID}")
fi
# [ "${nmip}" != "" ] && crontab -l | sshpass -p${1} ssh -o 'StrictHostKeyChecking no' nm_crontab_user@${nmip} "cat > ${DS_CRONTAB_FOLDER}/${hostname}.crontab"
else
echo "No user password specified"
fi
}
FIXCRONTAB(){
crontab -l | grep -v 'dsmon hdd' | crontab -
crontab -l | grep -v 'dsmon sys' | crontab -
(crontab -l ; echo "$(($RANDOM % 59)) */1 * * * /usr/local/bin/dsmon hdd >/dev/null 2>&1")| crontab -
(crontab -l ; echo "*/5+$(($RANDOM % 4)) * * * * /usr/local/bin/dsmon sys >/dev/null 2>&1")| crontab -
crontab -l | grep "dsmon"
}
case $action in
check) CHECK ${2};;
setupssh) SETUPSSH;;
get-crontabs) GETCRONTABS ${2};;
get-crontab) GETCRONTAB ${2};;
fix-crontab) FIXCRONTAB;;
gettsip) GETTSIP;;
gettsip-node) GETTSIP_NODE;;
run) RUN ${2};;
runcmd) RUNCMD ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9} ${10} ${11};;
update);;
*) RUN ${action};;
esac
exit 0