#!/usr/bin/env bash # powerwall - CLI commands to control VM guest power source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/dsmon/config.settings.inc action="$1" RUN(){ if [ "${SERVERMON_ID}" != "" ]; then if [ "${1}" = "hdd" ]; then DRIVEINFO=$(df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|media-data|Software' | awk '{ print $1 " " $2 " " $4 }') DRIVEINFO=(${DRIVEINFO}) NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) DRIVE_INFO=() for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do ii=$((${i}*3)) ia=${DRIVEINFO[${ii}]} ib=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]//M/}/1024") ic=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]//M/}/1024") ia=${ia//\/dev\/mapper\//} ia=${ia//\/dev\//} if [ "${ia}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${ia}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true else #echo "${ia};${ib};${ic}" DRIVE_INFO+=("${ia};${ib};${ic}") fi done QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','hdd','${DRIVE_INFO[@]}');" MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 6033 -u sysmoninsert -e "${QRY}" elif [ "${1}" = "sys" ]; then MEMORY=$(free -m | awk 'NR==2{printf "%.2f\t\t", $3*100/$2 }') CPU=$(top -bn1 | grep load | awk '{printf "%.2f\t\t\n", $(NF-2)}') QRY="USE servermonitor; INSERT INTO sysinfo (\`host\`, \`entry\`, \`value\`) VALUES ('${SERVERMON_ID}','sys','${CPU// /};${MEMORY}');" MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "${QRY}" fi SERVERHOSTNAME=(`hostname`) SERVERIP=(`hostname -I`) SERVERIPS='' for nip in "${SERVERIP[@]}"; do if [[ "${nip}" == *"."* ]]; then if [ "${SERVERIPS}" = "" ]; then SERVERIPS=${nip} else SERVERIPS=${SERVERIPS}";"${nip} fi fi done thresholdlog=() for KEY in "${!THRESHOLD[@]}"; do thresholdlog+=("${KEY}:${THRESHOLD[$KEY]}") done QRY="USE servermonitor; UPDATE hosts SET ip='${SERVERIPS// /}',hostname='${SERVERHOSTNAME}',limits='${thresholdlog[@]}' WHERE id='${SERVERMON_ID}';" MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "${QRY}" fi } CHECK(){ start=`date +%s` declare -A host_ip declare -A host_name declare -A host_limits declare -A host_ids declare -a hostnames_sort while read hostid hostname hostip hostlimits; do if [ "$hostid" != "id" ]; then hostip=$(echo $hostip | cut -d ";" -f1) # echo "$hostid - $hostip - $hostname - $hostlimits" host_ip[${hostid}]=$hostip host_name[${hostid}]=$hostname host_limits[${hostid}]=$hostlimits host_ids[$hostname]=$hostid hostnames_sort+=("$hostname") fi done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts WHERE disabled=0") IFS=$'\n' hostnames_sort=($(sort <<<"${hostnames_sort[*]}")); unset IFS #for hostid in "${!host_ip[@]}"; do for hostname in "${hostnames_sort[@]}"; do hostid=${host_ids[$hostname]} if [ "${1}" != "report" ]; then echo -e "${idsST[Bold]}"; DIVIDER false lightCyan 85 echo -e "${idsCL[LightCyan]} ${host_name[$hostid]} - Drive Space Check${idsCL[Default]}" DIVIDER false lightCyan 85 fi checkhost=$(CHECK_HOST ${host_ip[$hostid]}) if [ "${checkhost}" != "false" ]; then declare -A host_limits_tmp hostlimits=(${host_limits[${hostid}]}) for hl in ${hostlimits[@]}; do hlname=$(echo $hl | cut -d ":" -f1) hllim=$(echo $hl | cut -d ":" -f2) host_limits_tmp[$hlname]=$hllim done DRIVEINFO=$(ssh root@${host_ip[$hostid]} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|media-data|Software' | awk '{ print $1 " " $2 " " $4 }') DRIVEINFO=(${DRIVEINFO}) # for x in "${!DRIVEINFO[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO[$x]}" ; done NUMDRIVES=$((${#DRIVEINFO[@]} / 3)) declare -A DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME declare -a DRIVES for ((i = 0 ; i <= $((${NUMDRIVES}-1)) ; i++)); do ii=$((${i}*3)) dname=${DRIVEINFO[${ii}]} dname=${dname//\/dev\/mapper\//} dname=${dname//\/dev\//} dname_short=${dname#*vg-} [ "$dname_short" = "" ] && dname_short=$dname if [ "${dname}" = "sda1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "sda2" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "shm" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "nvme0n1p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true elif [ "${dname}" = "mmcblk0p1" ] && [ ${DRIVEINFO[$((${ii}+1))]//M/} -lt 1024 ]; then systempartition=true else dtot=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+1))]}/1024/10") dfree=$(bc <<< "scale=2; ${DRIVEINFO[$((${ii}+2))]}/1024/10") dfreeper=$(printf "%0.2f" $(jq -n ${dfree}/${dtot}*100)) DRIVEINFO_TOT[$dname]=$dtot DRIVEINFO_FREE[$dname]=$dfree DRIVEINFO_FREEPER[$dname]=$dfreeper DRIVEINFO_SHORTNAME[$dname]=$dname_short DRIVES+=("$dname") fi done IFS=$'\n' DRIVES=($(sort <<<"${DRIVES[*]}")); unset IFS # for x in "${!DRIVEINFO_TOT[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_TOT[$x]}" ; done # echo # for x in "${!DRIVEINFO_FREE[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREE[$x]}" ; done # echo # for x in "${!DRIVEINFO_FREEPER[@]}"; do printf "[%s]=%s\n" "$x" "${DRIVEINFO_FREEPER[$x]}" ; done # echo if [ "${1}" != "report" ]; then c=0; cw=20; spcA='' spc1=`expr ${cw} - 7` until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done c=0; cw=10; spcB='' spc2=`expr ${cw} - 4` until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done c=0; cw=10; spcC='' spc3=`expr ${cw} - 5` until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done c=0; cw=16; spcD='' spc4=`expr ${cw} - 3` until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done echo -e "${idsST[Reset]}${idsCL[White]} DRIVE${spcA} FREE GB${spcB}FREE %${spcC}TOTAL GB${spcD}WARN % / CRIT %${idsCL[Default]}" fi for DRIVE in "${DRIVES[@]}"; do [ "${host_limits_tmp[$DRIVE]}" = "" ] && host_limits_tmp[$DRIVE]=81 WARNING_LEVEL=$(printf "%0.2f" $(jq -n 100-${host_limits_tmp[$DRIVE]})) CRITICAL_LEVEL=$(printf "%0.2f" $(jq -n $WARNING_LEVEL-$WARNING_LEVEL/2)) if [ "${1}" != "report" ]; then c=0; cw=20; spcA='' spc1=`expr ${cw} - ${#DRIVEINFO_SHORTNAME[$DRIVE]}` until [ $c = ${spc1} ]; do spcA="${spcA} "; c=`expr $c + 1`; done c=0; cw=10; spcB='' spc2=`expr ${cw} - ${#DRIVEINFO_FREE[$DRIVE]}` until [ $c = ${spc2} ]; do spcB="${spcB} "; c=`expr $c + 1`; done c=0; cw=10; spcC='' spc3=`expr ${cw} - ${#DRIVEINFO_FREEPER[$DRIVE]}` until [ $c = ${spc3} ]; do spcC="${spcC} "; c=`expr $c + 1`; done c=0; cw=16; spcD='' spc4=`expr ${cw} - ${#DRIVEINFO_TOT[$DRIVE]}` until [ $c = ${spc4} ]; do spcD="${spcD} "; c=`expr $c + 1`; done fi if (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${CRITICAL_LEVEL}") )); then fs_status='CRITICAL' fs_status_color='RedBold' elif (( $(bc <<<"${DRIVEINFO_FREEPER[$DRIVE]} <= ${WARNING_LEVEL}") )); then fs_status='Warning' fs_status_color='Yellow' else fs_status='Normal' fs_status_color='Green' fi [ "${1}" != "report" ] && echo -e "${idsCL[Cyan]} ${DRIVEINFO_SHORTNAME[$DRIVE]}${spcA}${idsCL[$fs_status_color]}${DRIVEINFO_FREE[$DRIVE]} GB${spcB}${DRIVEINFO_FREEPER[$DRIVE]}%${spcC}${DRIVEINFO_TOT[$DRIVE]} GB${idsST[Reset]}${spcD}${idsCL[Cyan]}( ${idsCL[Yellow]}${WARNING_LEVEL}%${idsCL[Cyan]} / ${idsCL[RedBold]}${CRITICAL_LEVEL}%${idsST[Reset]}${idsCL[Cyan]} )${idsCL[Default]}" if [ "${ALERT_EMAIL}" != "" ] && [ "${fs_status}" != "Normal" ] && [ "${1}" = "report" ]; then echo -e "${host_name[$hostid]} - (${host_ip[$hostid]})\n\n${DRIVE} : ${DRIVEINFO_FREE[$DRIVE]}GB out of ${DRIVEINFO_TOT[$DRIVE]}GB Free (${DRIVEINFO_FREEPER[$DRIVE]}%)\n\n$(date)" | mail -s "Free Space ${fs_status}: '${host_name[$hostid]}'" ${ALERT_EMAIL} fi done unset DRIVEINFO_TOT DRIVEINFO_FREE DRIVEINFO_FREEPER DRIVEINFO_SHORTNAME DRIVES DRIVEINFO host_limits_tmp else [ "${1}" != "report" ] && echo -e "${idsCL[Yellow]} ${host_name[$hostid]} is down${idsCL[Default]}"; echo fi done end=`date +%s` runtime=$((end-start)) echo "runtime: ${runtime}" } SETUPSSH(){ declare -A host_ip declare -A host_name while read hostid hostname hostip hostlimits; do if [ "$hostid" != "id" ]; then host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1) host_name[${hostid}]=$hostname fi done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts") for hostid in "${!host_ip[@]}"; do echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - SSH KEY COPY${idsCL[Default]}${idsST[Reset]}" ssh-copy-id root@${host_ip[$hostid]} done } RUNCMD(){ declare -A host_ip declare -A host_name # while read hostid hostname hostip hostlimits; do # if [ "$hostid" != "id" ]; then # host_ip[${hostid}]=$(echo $hostip | cut -d ";" -f1) # host_name[${hostid}]=$hostname # fi # done <<< $(MYSQL_PWD="sysmoninsert" mysql -h mysqldb.scity.us -P 3306 -u sysmoninsert -e "SELECT id,host,ip,limits FROM servermonitor.hosts") # # for hostid in "${!host_ip[@]}"; do # echo -e "${idsST[Bold]}${idsCL[LightCyan]} ${host_name[$hostid]} - Running command: ${1}${idsCL[Default]}${idsST[Reset]}" # ssh root@${host_ip[$hostid]} "${1}" # done echo ${1} } case $action in check) CHECK ${2};; setupssh) SETUPSSH;; run) RUN ${2};; runcmd) RUNCMD "${2}";; update);; *) RUN ${action};; esac exit 0