270 lines
9.4 KiB
Bash
Executable File
270 lines
9.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# powerwall - CLI commands to control VM guest power
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/dsmon/config.settings.inc
|
|
|
|
action="$1"
|
|
|
|
RUN(){
|
|
if [ "${SERVERMON_ID}" != "" ]; then
|
|
|
|
if [ "${1}" = "hdd" ]; then
|
|
DRIVEINFO=`df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage' | awk '{ print $1 " " $2 " " $4 }'`
|
|
DRIVEINFO=(${DRIVEINFO})
|
|
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
|
|
|
DRIVE_INFO=()
|
|
|
|
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
|
ii=$((${i}*3))
|
|
ia=${DRIVEINFO[${ii}]}
|
|
ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024")
|
|
ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024")
|
|
ia=${ia//\/dev\/mapper\//}
|
|
ia=${ia//\/dev\//}
|
|
|
|
if [ "${ia}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${ia}" != "overlay" ]; then
|
|
#echo "${ia};${ib};${ic}"
|
|
DRIVE_INFO+=("${ia};${ib};${ic}")
|
|
|
|
fi
|
|
|
|
done
|
|
|
|
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');"
|
|
MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 6033 -u sysmoninsert -e "${QRY}"
|
|
|
|
elif [ "${1}" = "sys" ]; then
|
|
|
|
MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }')
|
|
CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}')
|
|
QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU// /};${MEMORY}');"
|
|
MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "${QRY}"
|
|
fi
|
|
|
|
SERVERHOSTNAME=(`hostname`)
|
|
SERVERIP=(`hostname -I`)
|
|
SERVERIPS=''
|
|
for nip in "${SERVERIP[@]}"; do
|
|
if [[ "${nip}" == *"."* ]]; then
|
|
if [ "${SERVERIPS}" = "" ]; then
|
|
SERVERIPS=${nip}
|
|
else
|
|
SERVERIPS=${SERVERIPS}";"${nip}
|
|
fi
|
|
fi
|
|
done
|
|
|
|
thresholdlog=()
|
|
for KEY in "${!THRESHOLD[@]}"; do
|
|
thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}")
|
|
done
|
|
|
|
QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';"
|
|
mysql -h mysqldb.scity.us -P 3306 --user=sysmoninsert --password=sysmoninsert -e "${QRY}"
|
|
|
|
|
|
fi
|
|
}
|
|
|
|
CHECK(){
|
|
start=`date +%s`
|
|
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
declare -A host_limits
|
|
declare -A host_ids
|
|
declare -a hostnames_sort
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
hostip=$(echo $hostip | cut -d ";" -f1)
|
|
# echo "$hostid - $hostip - $hostname - $hostlimits"
|
|
host_ip[${hostid}]=$hostip
|
|
host_name[${hostid}]=$hostname
|
|
host_limits[${hostid}]=$hostlimits
|
|
host_ids[$hostname]=$hostid
|
|
hostnames_sort+=("$hostname")
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0")
|
|
|
|
IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS
|
|
|
|
#for hostid in "${!host_ip[@]}"; do
|
|
for hostname in "${hostnames_sort[@]}"; do
|
|
hostid=${host_ids[$hostname]}
|
|
if [ "${1}" != "report" ]; then
|
|
echo -e "${idsST[Bold]}"; DIVIDER false lightCyan 100
|
|
echo -e "${idsCL[LightCyan]} ${host_name[$hostid]} - Drive Space Check${idsCL[Default]}"
|
|
DIVIDER false lightCyan 100
|
|
fi
|
|
|
|
checkhost=$(CHECK_HOST ${host_ip[$hostid]})
|
|
if [ "${checkhost}" != "false" ]; then
|
|
declare -A host_limits_tmp
|
|
hostlimits=(${host_limits[${hostid}]})
|
|
for hl in ${hostlimits[@]}; do
|
|
hlname=$(echo $hl | cut -d ":" -f1)
|
|
hllim=$(echo $hl | cut -d ":" -f2)
|
|
host_limits_tmp[$hlname]=$hllim
|
|
done
|
|
|
|
DRIVEINFO=$(ssh root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|media-data|Software' | awk '{ print $1 " " $2 " " $4 }')
|
|
DRIVEINFO=(${DRIVEINFO})
|
|
|
|
# for x in "${!DRIVEINFO[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO[$x]}" ; done
|
|
|
|
NUMDRIVES=$((${#DRIVEINFO[@]} / 3))
|
|
declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER
|
|
declare -a DRIVES
|
|
|
|
for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do
|
|
ii=$((${i}*3))
|
|
dname=${DRIVEINFO[${ii}]}
|
|
dname=${dname//\/dev\/mapper\//}
|
|
dname=${dname//\/dev\//}
|
|
|
|
if [ "${dname}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${dname}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${dname}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${dname}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
elif [ "${dname}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then
|
|
systempartition=true
|
|
else
|
|
dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10")
|
|
dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10")
|
|
dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100))
|
|
|
|
DRIVEINFO_TOT[$dname]=$dtot
|
|
DRIVEINFO_FREE[$dname]=$dfree
|
|
DRIVEINFO_FREEPER[$dname]=$dfreeper
|
|
DRIVES+=("$dname")
|
|
fi
|
|
done
|
|
|
|
IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS
|
|
|
|
# for x in "${!DRIVEINFO_TOT[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_TOT[$x]}" ; done
|
|
# echo
|
|
# for x in "${!DRIVEINFO_FREE[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREE[$x]}" ; done
|
|
# echo
|
|
# for x in "${!DRIVEINFO_FREEPER[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREEPER[$x]}" ; done
|
|
# echo
|
|
|
|
if [ "${1}" != "report" ]; then
|
|
c=0; cw=40; spcA=''
|
|
spc1=`expr ${cw} - 5`
|
|
until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcB=''
|
|
spc2=`expr ${cw} - 4`
|
|
until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcC=''
|
|
spc3=`expr ${cw} - 5`
|
|
until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
|
|
c=0; cw=12; spcD=''
|
|
spc4=`expr ${cw} - 3`
|
|
until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
|
|
echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA} FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN % / CRIT %${idsCL[Default]}"
|
|
fi
|
|
|
|
for DRIVE in "${DRIVES[@]}"; do
|
|
|
|
[ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81
|
|
|
|
|
|
|
|
WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]}))
|
|
CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2))
|
|
|
|
if [ "${1}" != "report" ]; then
|
|
c=0; cw=40; spcA=''
|
|
spc1=`expr ${cw} - ${#DRIVE}`
|
|
until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcB=''
|
|
spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}`
|
|
until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done
|
|
c=0; cw=10; spcC=''
|
|
spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}`
|
|
until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done
|
|
c=0; cw=12; spcD=''
|
|
spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}`
|
|
until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done
|
|
fi
|
|
|
|
if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then
|
|
fs_status='CRITICAL'
|
|
fs_status_color='RedBold'
|
|
|
|
elif (( $(bc <<<"$dfreeper <= ${WARNING_LEVEL}") )); then
|
|
fs_status='Warning'
|
|
fs_status_color='Yellow'
|
|
|
|
else
|
|
fs_status=''
|
|
fs_status_color='Green'
|
|
fi
|
|
|
|
[ "${1}" != "report" ] && echo -e "${idsCL[Cyan]} ${DRIVE}${spcA}:${idsCL[$fs_status_color]} ${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${idsST[Reset]}${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[Red]}${CRITICAL_LEVEL}%${idsCL[Cyan]} )${idsCL[Default]}"
|
|
if [ "${ALERT_EMAIL}" != "" ]; then echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)\n\n$(date)" | mail -s "Free Space ${fs_status}: '${host_name[$hostid]}'" ${ALERT_EMAIL}; fi
|
|
|
|
done
|
|
|
|
unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVES DRIVEINFO host_limits_tmp
|
|
|
|
else
|
|
[ "${1}" != "report" ] && echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}"; echo
|
|
fi
|
|
|
|
done
|
|
|
|
end=`date +%s`
|
|
runtime=$((end-start))
|
|
echo "runtime: ${runtime}"
|
|
}
|
|
|
|
SETUPSSH(){
|
|
declare -A host_ip
|
|
declare -A host_name
|
|
|
|
while read hostid hostname hostip hostlimits; do
|
|
if [ "$hostid" != "id" ]; then
|
|
host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1)
|
|
host_name[${hostid}]=$hostname
|
|
fi
|
|
done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts")
|
|
|
|
for hostid in "${!host_ip[@]}"; do
|
|
echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - SSH KEY COPY${idsCL[Default]}${idsST[Reset]}"
|
|
ssh-copy-id root@${host_ip[$hostid]}
|
|
done
|
|
}
|
|
|
|
|
|
case $action in
|
|
check) CHECK ${2};;
|
|
setupssh) SETUPSSH;;
|
|
run) RUN ${2};;
|
|
update);;
|
|
*) RUN ${action};;
|
|
esac
|
|
|
|
exit 0
|
|
|
|
|