#!/usr/bin/env bash # powerwall - CLI commands to control VM guest power source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/dsmon/config.settings.inc action="$1" RUN(){ if [ "${SERVERMON_ID}" != "" ]; then if [ "${1}" = "hdd" ]; then DRIVEINFO=`df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage' | awk '{ print $1 " " $2 " " $4 }'` DRIVEINFO=(${DRIVEINFO}) NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) DRIVE_INFO=() for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do ii=$((${i}*3)) ia=${DRIVEINFO[${ii}]} ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024") ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024") ia=${ia//\/dev\/mapper\//} ia=${ia//\/dev\//} if [ "${ia}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" != "overlay" ]; then #echo "${ia};${ib};${ic}" DRIVE_INFO+=("${ia};${ib};${ic}") fi done QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');" MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 6033 -u sysmoninsert -e "${QRY}" elif [ "${1}" = "sys" ]; then MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }') CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}') QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU// /};${MEMORY}');" MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "${QRY}" fi SERVERHOSTNAME=(`hostname`) SERVERIP=(`hostname -I`) SERVERIPS='' for nip in "${SERVERIP[@]}"; do if [[ "${nip}" == *"."* ]]; then if [ "${SERVERIPS}" = "" ]; then SERVERIPS=${nip} else SERVERIPS=${SERVERIPS}";"${nip} fi fi done thresholdlog=() for KEY in "${!THRESHOLD[@]}"; do thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}") done QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';" mysql -h mysqldb.scity.us -P 3306 --user=sysmoninsert --password=sysmoninsert -e "${QRY}" fi } CHECK(){ start=`date +%s` declare -A host_ip declare -A host_name declare -A host_limits declare -A host_ids declare -a hostnames_sort while read hostid hostname hostip hostlimits; do if [ "$hostid" != "id" ]; then hostip=$(echo $hostip | cut -d ";" -f1) # echo "$hostid - $hostip - $hostname - $hostlimits" host_ip[${hostid}]=$hostip host_name[${hostid}]=$hostname host_limits[${hostid}]=$hostlimits host_ids[$hostname]=$hostid hostnames_sort+=("$hostname") fi done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0") IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS #for hostid in "${!host_ip[@]}"; do for hostname in "${hostnames_sort[@]}"; do hostid=${host_ids[$hostname]} echo -e "${idsST[Bold]}"; DIVIDER false "" 100 echo -e "${idsCL[LightCyan]} ${host_name[$hostid]} - Drive Space Check${idsCL[Default]}" DIVIDER false "" 100; c=0; cw=40; spcA='' spc1=`expr ${cw} - 5` until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done c=0; cw=10; spcB='' spc2=`expr ${cw} - 4` until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done c=0; cw=10; spcC='' spc3=`expr ${cw} - 5` until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done c=0; cw=12; spcD='' spc4=`expr ${cw} - 1` until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA} FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN / CRIT${idsCL[Default]}" checkhost=$(CHECK_HOST ${host_ip[$hostid]}) if [ "${checkhost}" != "false" ]; then declare -A host_limits_tmp hostlimits=(${host_limits[${hostid}]}) for hl in ${hostlimits[@]}; do hlname=$(echo $hl | cut -d ":" -f1) hllim=$(echo $hl | cut -d ":" -f2) host_limits_tmp[$hlname]=$hllim done DRIVEINFO=$(ssh root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|media-data|Software' | awk '{ print $1 " " $2 " " $4 }') DRIVEINFO=(${DRIVEINFO}) # for x in "${!DRIVEINFO[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO[$x]}" ; done NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER declare -a DRIVES for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do ii=$((${i}*3)) dname=${DRIVEINFO[${ii}]} dname=${dname//\/dev\/mapper\//} dname=${dname//\/dev\//} if [ "${dname}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true else dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10") dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10") dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100)) DRIVEINFO_TOT[$dname]=$dtot DRIVEINFO_FREE[$dname]=$dfree DRIVEINFO_FREEPER[$dname]=$dfreeper DRIVES+=("$dname") fi done IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS # for x in "${!DRIVEINFO_TOT[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_TOT[$x]}" ; done # echo # for x in "${!DRIVEINFO_FREE[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREE[$x]}" ; done # echo # for x in "${!DRIVEINFO_FREEPER[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREEPER[$x]}" ; done # echo for DRIVE in "${DRIVES[@]}"; do [ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81 WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]})) CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2)) if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then fs_status='error' fs_status_color='Red' #if [ "${ALERT_EMAIL}" != "" ]; then echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]} GB free\t$(date)" | mail -s "Free Space Critical: '${host_name[$hostid]}'" ${ALERT_EMAIL}; fi elif (( $(bc <<<"$dfreeper <= ${WARNING_LEVEL}") )); then fs_status='warn' fs_status_color='Yellow' #if [ "${ALERT_EMAIL}" != "" ]; then echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]} GB free\t$(date)" | mail -s "Free Space Warning: '${host_name[$hostid]}'" ${ALERT_EMAIL}; fi else fs_status='' fs_status_color='Green' fi c=0; cw=40; spcA='' spc1=`expr ${cw} - ${#DRIVE}` until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done c=0; cw=10; spcB='' spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}` until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done c=0; cw=10; spcC='' spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}` until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done c=0; cw=12; spcD='' spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}` until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done echo -e "${idsCL[Cyan]} ${DRIVE}${spcA}:${idsCL[$fs_status_color]} ${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[Red]}${CRITICAL_LEVEL}%${idsCL[Cyan]} )${idsCL[Default]}" done unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVES DRIVEINFO host_limits_tmp else echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}" fi done end=`date +%s` runtime=$((end-start)) echo echo "runtime: ${runtime}" } SETUPSSH(){ declare -A host_ip declare -A host_name while read hostid hostname hostip hostlimits; do if [ "$hostid" != "id" ]; then host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1) host_name[${hostid}]=$hostname fi done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts") for hostid in "${!host_ip[@]}"; do echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - SSH KEY COPY${idsCL[Default]}${idsST[Reset]}" ssh-copy-id root@${host_ip[$hostid]} done } case $action in check) CHECK;; setupssh) SETUPSSH;; run) RUN ${2};; update);; *) RUN ${action};; esac exit 0