voltron/NodeMgmt
1
1
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
David Schroeder
2023-07-04 11:09:40 -05:00
parent f53920a042
commit 3f8e796124
6 changed files with 199 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
nodemgmt-scripts.sh
View File

@@ -40,14 +40,14 @@ CHECK_DOCKER_SERVICES(){
echo -e "${idsCL[LightCyan]} - ${uptime} ${idsCL[LightYello]}${LH}${idsCL[Default]}"
DIVIDER false green
fi
if [ -f ${TMPFOLDER}/${nip}.down ]; then
if [ -f ${TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
if [ -f ${NM_TMPFOLDER}/${nip}.down ]; then
if [ -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
rm -f ${TMPFOLDER}/${nip}.down
rm -f ${TMPFOLDER}/${nip}.errtime
rm -f ${NM_TMPFOLDER}/${nip}.down
rm -f ${NM_TMPFOLDER}/${nip}.errtime
SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]-UP" "${NM_HOSTNAMES[${nip}]}[${nip}] is back UP! It was down for $(SHOW_TIME ${toterrtime})"
fi
@@ -80,16 +80,16 @@ CHECK_DOCKER_SERVICES(){
fi
done
else
if [ ! -f ${TMPFOLDER}/${nip}.down ]; then
touch ${TMPFOLDER}/${nip}.down
if [ ! -f ${TMPFOLDER}/${nip}.errtime ]; then
touch ${TMPFOLDER}/${nip}.errtime
if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then
touch ${NM_TMPFOLDER}/${nip}.down
if [ ! -f ${NM_TMPFOLDER}/${nip}.errtime ]; then
touch ${NM_TMPFOLDER}/${nip}.errtime
fi
else
errtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.down)
errtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.down)
fi
if [ "${ST_ACTION}" != "check" ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${nip}.errtime)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${nip}.errtime)
echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Red]} - Node has been down for $(SHOW_TIME ${toterrtime}) ${idsCL[LightYello]}${LH}${idsCL[Default]}"
fi
fi
@@ -122,7 +122,7 @@ BACKUP(){
echo -e "${idsCL[Yellow]}Not found${idsCL[Default]}"
fi
done
touch ${FOLDER}/backup.lastrun
touch ${NM_FOLDER}/backup.lastrun
# if [ "${action}" != "backup" ]; then
echo
DIVIDER true
@@ -602,7 +602,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -623,7 +623,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -643,7 +643,7 @@ HAPROXY-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/haproxy.config > /etc/haproxy/haproxy.cfg"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -672,7 +672,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
for stc in "${!servicestocheck}"; do
if [ "$(systemctl is-active ${stc}.service)" != "active" ]; then
SENDNOTICE "Node-Balancing Notice" "Starting ${stc} on ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} (${local_ip}), stopping on the other Nodes"
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${LOGFILE}
echo "$(date) - ${NM_NODETYPES["WEB"]}-Node${local_ip: -1} - Starting ${stc} on ${local_ip}, stopping on the other Nodes" >> ${NM_LOGFILE}
[ "${stc}" = "pdnsadmin" ] && stc="pdnsadmin.socket pdnsadmin.service"
for nip in "${!hosts}"; do
@@ -712,7 +712,7 @@ SINGLE_SERVER_SERVICE_CHECK(){
fi
if [ "$(systemctl is-active nginx)" != "active" ]; then
${SCRIPT} service web restart ${local_ip}
${NM_SCRIPT} service web restart ${local_ip}
fi
fi
}
@@ -734,7 +734,7 @@ KEEPALIVE-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -755,7 +755,7 @@ KEEPALIVE-CONFIG() {
${NCMD} "nodemgmt update q"
${NCMD} "rm -f /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
${NCMD} "sed -e 's;%NIP%;${nip};g' ${NM_FOLDER}/templates/keepalived.config > /etc/keepalived/keepalived.conf"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
@@ -778,15 +778,15 @@ NODEUPDATE() {
declare -i cw; declare -i spc1; declare -i c
echo -e "${idsCL[LightGreen]}Updating All Nodes${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}-------------------------------------------${idsCL[Default]}"
nip=${NM_NODEMANAGER}
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nip}"* ]]; then
nip='localhost'
nmip=${NM_NODEMANAGER}
if [[ $(ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | head -n 1 | awk '/inet/ {print $2}' | cut -d'/' -f1) == *"${nmip}"* ]]; then
nmip='localhost'
fi
c=0; cw=24; spca=''; spc1=${cw}-7; until [ $c = ${spc1} ]; do spca="${spca} "; c=`expr $c + 1`; done
echo -en "Updating Node-Manager[$nip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
ssh -tq root@${nip} "nodemgmt update q"
echo -en "Updating Node-Manager[$nmip]${spca} ~ ${idsCL[LightCyan]}NodeMGMT${idsCL[Default]} : "
ssh -tq root@${nmip} "nodemgmt update q"
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
ssh -tq root@${nip} "runup runup"
ssh -tq root@${nmip} "runup runup"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
echo
for NTYPE in "${NODE_TYPES[@]}"; do
@@ -803,6 +803,8 @@ NODEUPDATE() {
${NCMD} "nodemgmt update q" >/dev/null 2>&1
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}LinUPx${idsCL[Default]} : "
${NCMD} "runup runup" >/dev/null 2>&1
echo -en "${idsCL[Green]}COMPLETE${idsCL[Default]} ~ ${idsCL[LightCyan]}Sync Defaults${idsCL[Default]} : "
ssh -tq root@${nmip} "scp ${NM_FOLDER}/defaults.local.inc root@${nip}:${NM_FOLDER}/"
echo -e "${idsCL[Green]}COMPLETE${idsCL[Default]}"
nid=`expr $nid + 1`
done
@@ -833,7 +835,7 @@ VCENTER-SSL(){
CHAIN=/root/.acme.sh/$VCHOSTNAME/fullchain.cer
echo -en "${idsCL[LightCyan]}Checking days left on vCenter cert... ${idsCL[Default]}"
VCCERTDAYS=$(${FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
VCCERTDAYS=$(${NM_FOLDER}/ssl-cert-check/ssl-cert-check -p 443 -s ${VCHOSTNAME} -N)
VCCERTDAYS=${VCCERTDAYS#*=}
if [ "${VCCERTDAYS}" -gt "29" ] && [ "${1}" != "force" ]; then
@@ -899,10 +901,10 @@ VCENTER-SSL(){
echo
}
ADD_LOGROTATE_CRONTAB(){
if ! crontab -l | grep -q "${FOLDER}/tmp-logrotate"; then
(crontab -l ; echo "0 */1 * * * logrotate -f ${FOLDER}/tmp-logrotate")| crontab -
if ! crontab -l | grep -q "${NM_FOLDER}/tmp-logrotate"; then
(crontab -l ; echo "0 */1 * * * logrotate -f ${NM_FOLDER}/tmp-logrotate")| crontab -
fi
/bin/chmod 0644 ${FOLDER}/tmp-logrotate
/bin/chmod 0644 ${NM_FOLDER}/tmp-logrotate
echo
echo -e "${idsCL[Green]}The crontab entry has been made${idsCL[Default]}"
echo
@@ -1005,7 +1007,7 @@ DOWNLIST(){
idsST=('')
fi
declare -i cw; declare -i spc1; declare -i c
DOWN_LIST=`find ${FOLDER} -name "*.down"`
DOWN_LIST=`find ${NM_FOLDER} -name "*.down"`
if [ -z $action ] || [ "${action}" = "gui" ]; then
DIVIDER true
@@ -1038,10 +1040,10 @@ DOWNLIST(){
[ "${NM_HOSTNAMES[${host}]}" != "" ] && itemname="${itemname}: ${NM_HOSTNAMES[${host}]}"
fi
if [ -f ${TMPFOLDER}/${ditem}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.errtime)
if [ -f ${NM_TMPFOLDER}/${ditem}.errtime ]; then
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.errtime)
else
toterrtime=`date +%s`-$(stat -c %Y ${TMPFOLDER}/${ditem}.down)
toterrtime=`date +%s`-$(stat -c %Y ${NM_TMPFOLDER}/${ditem}.down)
fi
c=0; spc=''
@@ -1182,8 +1184,8 @@ GUI(){
update) ;;
update-nodes)
if [ "${2}" == "report" ]; then
rm -f ${LOGFOLDER}/update-nodes.lastrun
NODEUPDATE ${2} > ${LOGFOLDER}/update-nodes.lastrun
rm -f ${NM_LOGFOLDER}/update-nodes.lastrun
NODEUPDATE ${2} > ${NM_LOGFOLDER}/update-nodes.lastrun
else
NODEUPDATE ${2}
fi
@@ -1192,44 +1194,44 @@ GUI(){
backup) BACKUP;;
downlist)
if [ "${2}" = "report" ]; then
if [ "$(find ${FOLDER} -name '*.down')" != "" ]; then
rm -f ${LOGFOLDER}/status-downlist.lastrun
DOWNLIST report >> ${LOGFOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${LOGFOLDER}/status-downlist.lastrun)"
if [ "$(find ${NM_FOLDER} -name '*.down')" != "" ]; then
rm -f ${NM_LOGFOLDER}/status-downlist.lastrun
DOWNLIST report >> ${NM_LOGFOLDER}/status-downlist.lastrun
STATUSLOG="$(cat ${NM_LOGFOLDER}/status-downlist.lastrun)"
SENDNOTICE "Downlist Report" "Downlist Report\n${STATUSLOG}"
fi
else
DOWNLIST ${2}
fi;;
status)
if [ -f "${FOLDER}/status-check.running" ]; then
fd=`stat --format=%Y "${FOLDER}/status-check.running"`
if [ -f "${NM_FOLDER}/status-check.running" ]; then
fd=`stat --format=%Y "${NM_FOLDER}/status-check.running"`
cbd=$((`date +%s` - 450))
[ ${fd} -lt ${cbd} ] && rm -f "${FOLDER}/status-check.running"
[ ${fd} -lt ${cbd} ] && rm -f "${NM_FOLDER}/status-check.running"
fi
if [ ! -f "${FOLDER}/status-check.running" ]; then
if [ ! -f "${NM_FOLDER}/status-check.running" ]; then
start_status=`date +%s`
touch "${FOLDER}/status-check.running"
touch "${NM_FOLDER}/status-check.running"
if [ "${2}" == "report" ]; then
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${LOGFILE}
echo -e "$(date) - Starting status check #############################\n" > ${FOLDER}/status-check.running
STATUS report ${3} >> ${FOLDER}/status-check.running
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${FOLDER}/status-check.running
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${LOGFILE}
rm -f ${FOLDER}/status-check.lastrun
mv ${FOLDER}/status-check.running ${FOLDER}/status-check.lastrun
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Starting Status Check" >> ${NM_LOGFILE}
echo -e "$(date) - Starting status check #############################\n" > ${NM_FOLDER}/status-check.running
STATUS report ${3} >> ${NM_FOLDER}/status-check.running
echo -e "\n\n$(date) - Finished status check #############################\n" >> ${NM_FOLDER}/status-check.running
echo "$(date) - LOCAL - ${NODE_HOSTNAME} - Completed Status Check" >> ${NM_LOGFILE}
rm -f ${NM_FOLDER}/status-check.lastrun
mv ${NM_FOLDER}/status-check.running ${NM_FOLDER}/status-check.lastrun
end_status=`date +%s`
runtime_status=$((end_status-start_status))
echo -e "\nRuntime: ${runtime_status}" >> ${FOLDER}/status-check.lastrun
echo -e "\nRuntime: ${runtime_status}" >> ${NM_FOLDER}/status-check.lastrun
if [ "${3}" = "email" ]; then
echo -e "\n\n" >> ${FOLDER}/status-check.lastrun
DOWNLIST report >> ${FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${FOLDER}/status-check.lastrun)"
echo -e "\n\n" >> ${NM_FOLDER}/status-check.lastrun
DOWNLIST report >> ${NM_FOLDER}/status-check.lastrun
STATUSLOG="$(cat ${NM_FOLDER}/status-check.lastrun)"
SENDNOTICE "Status Report" "Status Report\n${STATUSLOG}"
fi
else
STATUS ${2} ${3}
rm -f ${FOLDER}/status-check.running
rm -f ${NM_FOLDER}/status-check.running
fi
else
# echo -e "${idsCL[LightYellow]}Status check already running, please wait and try again${idsCL[Default]}"
@@ -1237,8 +1239,8 @@ GUI(){
echo -en "${idsCL[LightCyan]}View the (L)ive log, or the (P)revious run log instead? ${idsCL[Default]}"
read -n1 choice
case $choice in
[Ll]) watch -n1 tail -n35 ${FOLDER}/status-check.running; echo;;
[Pp]) echo; cat ${FOLDER}/status-check.lastrun;;
[Ll]) watch -n1 tail -n35 ${NM_FOLDER}/status-check.running; echo;;
[Pp]) echo; cat ${NM_FOLDER}/status-check.lastrun;;
esac
echo
fi;;