From caec4404a6fca941171f69fd1b6b3b43ad70c240 Mon Sep 17 00:00:00 2001 From: David Schroeder Date: Sun, 1 Jun 2025 12:29:55 -0500 Subject: [PATCH] update --- defaults.inc | 3 ++- inc/certs.inc | 4 +-- inc/services.inc | 18 ++++++------- inc/sites.inc | 30 +++++++++++----------- inc/status.inc | 55 ++++++++++++++++++++++++++-------------- inc/vCenter-SSL.ps1 | 2 +- nodemgmt-scripts.sh | 62 ++++++++++++++++++++++----------------------- 7 files changed, 96 insertions(+), 78 deletions(-) diff --git a/defaults.inc b/defaults.inc index 5f29ff8f..7dd1d408 100755 --- a/defaults.inc +++ b/defaults.inc @@ -1,10 +1,11 @@ #!/usr/bin/env bash -VERS='5.5.9-05302025' +VERS='5.5.10-06012025' NM_BETA=false noheader=' service status-check nightlyrephp7.3-fpm,new backup report check checkcerts gitea update-nodes copynpmcerts singleservercheck update-dyndns backup-offsitepfsense gui nightlyreview update log betacheck offsite-staticroute-fix ' CERT_DAEMON='/opt/certbot/bin/certbot' +SSHCMD='/bin/ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1' declare -i errtime [ "${NM_FOLDER}" == "" ] && NM_FOLDER=/opt/idssys/nodemgmt diff --git a/inc/certs.inc b/inc/certs.inc index 66917541..9a2d7e8f 100755 --- a/inc/certs.inc +++ b/inc/certs.inc @@ -223,7 +223,7 @@ LISTCERTS(){ fi # if [ ! -z ${LOCAL_SERVICES+x} ]; then - # NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_HOSTS['LB'][0]}" + # NCMD="${SSHCMD} root@${NM_HOSTS['LB'][0]}" # #${NCMD} rm -f /tmp/ssllist # #${NCMD} 'for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done' > /tmp/ssllist # else @@ -492,7 +492,7 @@ LISTCERTS_NPM(){ done unset IFS if [ ! -z ${LOCAL_SERVICES+x} ]; then - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${MYSQL_HOSTS[0]}" + NCMD="${SSHCMD} root@${MYSQL_HOSTS[0]}" ${NCMD} rm -f /tmp/ssllist ${NCMD} 'for certdir in ${NM_CERTPATH}/live/*/ ; do echo $certdir; done' > /tmp/ssllist else diff --git a/inc/services.inc b/inc/services.inc index c82352c3..17e3e4e8 100755 --- a/inc/services.inc +++ b/inc/services.inc @@ -30,8 +30,8 @@ SINGLE_SERVER_SERVICE_CHECK(){ for docker in "${!dockerstocheck}"; do for nip in "${!hosts}"; do if [[ "${RUN_NODE_IP}" != *"${nip}"* ]] && [ "$(CHECK_HOST ${nip})" != "false" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1 - [ "${docker}" == "authelia" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1 + ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name='${docker}')" ] && docker stop '${docker} >/dev/null 2>&1 + [ "${docker}" == "authelia" ] && ${SSHCMD} root@${nip} '[ "$(docker ps -q -f name=auth_redis)" ] && docker stop auth_redis' >/dev/null 2>&1 fi done done @@ -161,7 +161,7 @@ SERVICE_MGMT(){ nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi [ ${QUIET} -eq 0 ] && echo -en "${NM_SRVCOPTS[${SERVICE['action']}]}ing on '$nip'... ${idsCL[Default]}" @@ -195,7 +195,7 @@ SERVICE_MGMT(){ nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi if [ ${QUIET} -eq 0 ]; then @@ -276,7 +276,7 @@ SERVICE_MGMT(){ if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then PH_CMD="" else - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!NT_HOST}" + PH_CMD="${SSHCMD} root@${!NT_HOST}" fi ${PH_CMD} touch ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl # ${PH_CMD} "echo -e \"Service ${SERVICE['service']} ${SERVICE['action']}\" >> ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl" @@ -285,7 +285,7 @@ SERVICE_MGMT(){ if [ "${checkhost}" != "false" ] && [ "${nip}" != "${!NT_HOST}" ]; then checked=false until [ "${checked}" = "true" ]; do - checked="`${PH_CMD} ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl)) -le 90 ] && echo true || echo false`" + checked="`${PH_CMD} ${SSHCMD} root@${nip} [ -f ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl ] && [ $(expr $(date +%s) - $(stat -L --format %Y ${NM_REPL_NGINX_PATHS[${RUN_NODE_TYPE}]}/.test.repl)) -le 90 ] && echo true || echo false`" done fi done @@ -310,7 +310,7 @@ SERVICE_MGMT(){ if [[ "${RUN_NODE_IP}" == *"${!NT_HOST}"* ]]; then PH_CMD="" else - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!NT_HOST}" + PH_CMD="${SSHCMD} root@${!NT_HOST}" fi ${PH_CMD} touch ${NM_REPL_CHECK_LOC[${SERVICE['service']}]}/.test.repl for nip in "${!NT_HOSTS}"; do @@ -363,7 +363,7 @@ SERVICE_MGMT(){ nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi NOGOCHK=true; @@ -386,7 +386,7 @@ SERVICE_MGMT(){ if [ "${SERVICE['service']}"= = "offsite-power-check" ]; then if [ ${QUIET} -eq 0 ] && [ "${SERVICE['action']}" != "status" ]; then - if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then + if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then echo -e "${idsCL[Green]}Done${idsCL[Default]}" else echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" diff --git a/inc/sites.inc b/inc/sites.inc index 7d040225..d2b4888c 100755 --- a/inc/sites.inc +++ b/inc/sites.inc @@ -54,7 +54,7 @@ DELSITE(){ echo fi - [ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${DEL_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml + [ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${DEL_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml SERVICE_MGMT nginx restart @@ -139,7 +139,7 @@ NEWSITE(){ exit 0 elif [[ ${overwrite} =~ ^[Yy]$ ]]; then rm -f ${nginxconfig} >/dev/null 2>&1 - [ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${MAIN_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml + [ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${MAIN_SITE}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml else exit 0 fi @@ -282,12 +282,12 @@ NEWSITE(){ echo -e "${idsCL[LightGreen]}Configuring Authelia SSO for '${idsCL[Yellow]}${MAIN_SITE}${idsCL[LightGreen]}' {${NGINX_SERVERNAME}}...${idsCL[Default]}" if [ "${SECURE}" == "2FA" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" else - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" fi - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1" fi ######################################### LOCAL @@ -474,20 +474,20 @@ NEWPROXYSITE_CREATE(){ if [[ "${SECURE}" = *"FA"* ]]; then echo -en "${idsCL[LightCyan]}Enabling ${SECURE} SSO access ... " sed -i "s/#include conf.d\/include\/secure-access.conf/include conf.d\/include\/secure-access.conf/g" ${nginxconfig} - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml + ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml if [ "${SECURE}" == "2FA" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies needing 2 factor below/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" else - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${MAIN_SITE}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" fi - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1" + ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + ${SSHCMD} root@${NM_AUTHELIA_IP} "/usr/bin/docker restart authelia >/dev/null 2>&1" echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}" echo else echo -en "${idsCL[LightCyan]}Disabling SSO access ... " sed -i "s/include conf.d\/include\/secure-access.conf/#include conf.d\/include\/secure-access.conf/g" ${nginxconfig} - [ "${NM_AUTHELIA_IP}" != "" ] && ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml + [ "${NM_AUTHELIA_IP}" != "" ] && ${SSHCMD} root@${NM_AUTHELIA_IP} sed -i "/${SITENAME}/d" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml echo -e "${idsCL[LightGreen]}Done${idsCL[Default]}" echo fi @@ -557,7 +557,7 @@ SITEINFO(){ done if [ "${onefacline}" == "" ]; then - [ "$(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1)" != "${NM_AUTHELIA_IP}" ] && ACMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP}" || ACMD="" + [ "$(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1)" != "${NM_AUTHELIA_IP}" ] && ACMD="${SSHCMD} root@${NM_AUTHELIA_IP}" || ACMD="" autheliaconfig=$(${ACMD} cat ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml) onefacline=$(echo "${autheliaconfig}" | grep -Fn one_factor | sort | tail -n1) onefacline=${onefacline%%:*} @@ -627,8 +627,8 @@ SITEINFO(){ sitefacline=${sitefacline%%:*} if [ "${sitefacline}" == "" ]; then SECURE='error' - # ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${SITENAME}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" - # ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + # ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -ie \"/domain: # Proxies only requiring username and password/a ~~~ - \\\"${SITENAME}\\\"\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" + # ${SSHCMD} root@${NM_AUTHELIA_IP} "sed -i \"s/~~~/ /g\" ${NM_DOCKER_COMPOSE_LOC['authelia']}/config/configuration.yml" elif [ ${sitefacline} -lt ${onefacline} ]; then SECURE='1FA' elif [ ${sitefacline} -lt ${twofacline} ]; then diff --git a/inc/status.inc b/inc/status.inc index bb319bf9..e1301091 100755 --- a/inc/status.inc +++ b/inc/status.inc @@ -68,7 +68,7 @@ STATUS(){ fi if [ ${skip} -eq 0 ]; then - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then var=${NTYPE}_HOSTS[@] @@ -237,7 +237,7 @@ STATUS(){ PH=${NODETYPE}_HOSTS[1] fi fi - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then var=${NTYPE}_HOSTS[@] @@ -306,7 +306,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ fi rm -f ${NM_TMPFOLDER}/${nip}~${rcheck}.repl* fi - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' & + ${SSHCMD} root@${nip} 'find ${NM_REPL_CHECK_LOC[${rcheck}]} -iname "*.sync-conflict-*" -exec rm {} \;' & elif [ "${checked}" == "skip" ]; then [ "${STATUS_ACTION}" != "check" ] && echo -e "${idsCL[LightCyan]}Skipped${idsCL[Default]}" @@ -344,7 +344,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ fi fi if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then if [ "${PH_CMD}" == "" ]; then @@ -377,7 +377,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ if [ ! -f ${NM_FOLDER}/${!PH}.down ] && [ ${skip} -eq 0 ]; then var=${NTYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" for rcheck in "${!var}"; do ${PH_CMD} rm -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.2023*.repl if [ $(${PH_CMD} ls ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.*.repl 2>/dev/null | wc -l) -gt 0 ]; then @@ -413,7 +413,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ var=${NTYPE}_HOSTS[@] for nip in "${!var}"; do if [[ $"{RNIP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost' - else NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH='' + else NCMD="${SSHCMD} root@${nip}"; LH='' fi if [ "${STATUS_ACTION}" != "check" ]; then echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]} [${nip}]${idsST[Reset]}${idsCL[LightCyan]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}" @@ -422,7 +422,7 @@ It was down for $(SHOW_TIME $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${ [ "${STATUS_ACTION}" != "check" ] && [ "${2}" != "report" ] && echo -en " ${idsCL[LightCyan]}Getting drives from server ... ${idsCL[Default]}" declare -A partitions - DRIVEINFO=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') + DRIVEINFO=$(${SSHCMD} root@${nip} df -BM | grep -vE '^Filesystem|tmpfs|cdrom|@|ram|loop|udev|veeamimage|nvme|localhost|shm|mmcblk|overlay|-volume|Music|Software' | awk '{ print $1 " " $2 " " $4 }') DRIVEINFO=(${DRIVEINFO}) if [ "${2}" != "report" ]; then echo -en "\e[1A" @@ -513,7 +513,7 @@ STATUS_NODE(){ if [[ "${RUN_NODE_IP}" == *"${nip}"* ]]; then NCMD=''; LH='- localhost' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}"; LH='' + NCMD="${SSHCMD} root@${nip}"; LH='' fi if [ "${NCMD}" != "" ] && [ ! -f ${STATUSRUN_TMPFOLDER}/.${nip}.* ]; then checkhost=$(CHECK_HOST ${nip}) @@ -617,7 +617,7 @@ STATUS_NODE(){ fi sr=0 - [ "${NODETYPE}" == "MM" ] && mmstop=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi") + [ "${NODETYPE}" == "MM" ] && mmstop=$(${SSHCMD} root@${nip} "if [ -f /opt/idssys/mediamanager/.tmp/mm.stop ] || [ -f /opt/idssys/mediamanager/.tmp/mm.noservices ]; then echo 1; fi") for srvc in "${!srvcs}"; do if [ "${STATUS_ACTION}" != "report" ]; then @@ -629,7 +629,7 @@ STATUS_NODE(){ fi tmp=${NM_SINGLESRVR_SERVICES[${NODETYPE}]} SSCHK=true - if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then + if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then SSCHK=skip elif [[ ${tmp[@]} =~ ${srvc} ]]; then if [[ $(${NCMD} ip addr show $(ip route | awk '/default/ { print $5 }') | grep "inet" | awk '/inet/ {print $2}' | cut -d'/' -f1) != *${NM_SINGLESRVR_IP[${NODETYPE}]}* ]]; then @@ -658,7 +658,7 @@ STATUS_NODE(){ mysqlgo=true fi else - if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then + if [ "$(${SSHCMD} root@${nip} ps -U root | grep "offsite-power-check.sh start" | grep -v "grep" | awk '{print $1}')" != "" ]; then srvctest=active else srvctest=notactive @@ -684,16 +684,33 @@ STATUS_NODE(){ touch ${NM_TMPFOLDER}/${nip}~${srvc}.sent fi fi + + + ######################################### + ############ SERVICE REPAIRS ############ + if [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.disable ] && [ ! -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ] && [ $(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) -gt 120 ]; then + if [ "${srvc}" == "pihole-FTL" ]; then + ${SSHCMD} root@${nip} 'pihole -r' & >/dev/null 2>&1 + touch ${NM_TMPFOLDER}/${nip}~${srvc}.fix + fi + fi + ######################################### + ######################################### + fi else [ "${STATUS_ACTION}" != "check" ] && echo -en "${idsCL[LightGreen]}Running" if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.down ]; then if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.sent ]; then [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.errtime ] && toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.errtime)) || toterrtime=$(expr $(date +%s) - $(stat -c %Y ${NM_TMPFOLDER}/${nip}~${srvc}.down)) - SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})" - echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + if [ -f ${NM_TMPFOLDER}/${nip}~${srvc}.fix ]; then + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - was FIXED! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} was FIXED, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + else + SENDNOTICE "${NM_HOSTNAMES[${nip}]}[${nip}]" "'${srvc}' ${NM_SERVICE_DESC[${srvc}]} - is back UP! It was down for $(SHOW_TIME ${toterrtime})" + echo "$(date +%Y-%m-%d-%H-%M-%S) - ${nip} - ${NM_HOSTNAMES[${nip}]}[${nip}] - ${NM_SERVICE_DESC[${srvc}]} is back up, it was down for $(SHOW_TIME ${toterrtime})" >> ${NM_LOGFILE} + fi fi - fi if [[ "${NM_SINGLESRVR_SERVICES[${NODETYPE}]}" = *"${srvc}"* ]]; then for tnip in "${!NTHOSTS}"; do @@ -741,7 +758,7 @@ STATUS_NODE(){ fi tmp=${NM_SINGLESRVR_DOCKERS[${NODETYPE}]} SDCHK=true - if ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then + if ${SSHCMD} root@${nip} [ -f ${NM_TMPFOLDER}/.skip ]; then SDCHK=skip elif ([ "${NM_HOSTNAMES[${nip}]}" == "ESPHome" ] && [ "${docker}" == "portainer_agent" ]); then SDCHK=skip @@ -934,9 +951,9 @@ REPLCHECK(){ checked=false until [ "${checked}" == "" ]; do if [ "${PH_CMD}" == "" ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl` + ${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked=`${SSHCMD} root@${nip} "cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl` else - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`" + ${SSHCMD} root@${nip} [[ -f ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl ]] && checked="`${PH_CMD} \"${SSHCMD} root@${nip} \"cat ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\" | diff - ${NM_REPL_CHECK_LOC[${rcheck}]}/.test.${STATUS_START//-/}.repl\"`" fi if [ "`date +%s`" -gt "${timeout}" ]; then timeout=true @@ -969,7 +986,7 @@ REPLSTART(){ fi if [ ${skip} -eq 0 ] && [ "${!PH}" != "" ]; then - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" var=${NODETYPE}_REPL_CHECK[@] if [ ! -z ${!var+x} ]; then for rcheck in "${!var}"; do @@ -998,7 +1015,7 @@ REMOVE_CONFLICT_SYNC_FILES(){ for nip in "${!var}"; do if [ ! -f ${NM_TMPFOLDER}/${nip}.down ]; then echo -e "${idsCL[LightYellow]}Removing all Syncthing 'sync-conflict' files from ${idsCL[LightGreen]}${NM_HOSTNAMES[${nip}]}${idsCL[LightYellow]} [${nip}]${idsCL[Default]}" - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + PH_CMD="${SSHCMD} root@${nip}" var2=${NTYPE}_REPL_CHECK[@] for rcheck in "${!var2}"; do echo -en "Removing files from ${idsCL[LightCyan]}${NM_REPL_CHECK_LOC[${rcheck}]}/* ${idsCL[Default]}... " diff --git a/inc/vCenter-SSL.ps1 b/inc/vCenter-SSL.ps1 index 7fbec491..6d77491c 100644 --- a/inc/vCenter-SSL.ps1 +++ b/inc/vCenter-SSL.ps1 @@ -242,6 +242,6 @@ else { Write-Error "Failed to Replace Certificate, Please verify Correct Configuration and Retry" } -ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 $VEEAMHOSTSSH "Rescan-VBREntity -AllHosts" +${SSHCMD} $VEEAMHOSTSSH "Rescan-VBREntity -AllHosts" } diff --git a/nodemgmt-scripts.sh b/nodemgmt-scripts.sh index 6e55d365..4a380ed0 100755 --- a/nodemgmt-scripts.sh +++ b/nodemgmt-scripts.sh @@ -29,10 +29,10 @@ OFFSITE_STATICROUTE_FIX(){ if [ "$(CHECK_HOST_QC 10.10.0.1)" != "true" ]; then if [ "$(CHECK_HOST_QC ${NM_OFFSITE_GW_IP})" != "false" ]; then if [ "$(CHECK_HOST_QC 10.5.0.1)" != "false" ]; then - STSVPN_ADAPTER=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r | grep '10.5.0.0/20' | awk '{print $7}') - if [ "$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r | grep 10.10.0.0)" = "" ] && [ "${STSVPN_ADAPTER}" != "" ]; then - RUN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r add 10.6.1.0/24 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null) - RUN=$(ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} ip r add 10.10.0.0/16 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null) + STSVPN_ADAPTER=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r | grep '10.5.0.0/20' | awk '{print $7}') + if [ "$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r | grep 10.10.0.0)" = "" ] && [ "${STSVPN_ADAPTER}" != "" ]; then + RUN=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r add 10.6.1.0/24 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null) + RUN=$(${SSHCMD} root@${NM_OFFSITE_GW_IP} ip r add 10.10.0.0/16 dev ${STSVPN_ADAPTER} 2>&1 >/dev/null) if [ "${RUN}" == "" ]; then SENDNOTICE "Static Route to Homesite Missing" "Re-added the static route onto the Offsite UniFi Gateway" else @@ -100,7 +100,7 @@ SET-PERMISSIONS(){ if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi if [ "${2}" != "q" ]; then echo -en "${idsCL[Yellow]}Resetting folder permissions for Gitea on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}" @@ -123,7 +123,7 @@ SET-PERMISSIONS(){ if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi if [ "${2}" != "q" ]; then echo -en "${idsCL[Yellow]}Resetting folder permissions for PowerDNS-Admin on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}" @@ -144,7 +144,7 @@ SET-PERMISSIONS(){ if [[ ${RUN_NODE_IP} == *"${nip}"* ]]; then NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi if [ "${2}" != "q" ]; then echo -en "${idsCL[Yellow]}Resetting folder permissions for Nextcloud on ${NM_HOSTNAMES[${nip}]} (${nip})... ${idsCL[Default]}" @@ -174,7 +174,7 @@ SET-PERMISSIONS(){ nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -e "${idsCL[Yellow]}Setting folder permissions for ${NM_HOSTNAMES[${nip}]} (${nip})${idsCL[Default]}" echo -en "${idsCL[Cyan]}LetsEncrypt Certs ${idsCL[Default]}" @@ -193,7 +193,7 @@ SET-PERMISSIONS(){ nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi if [ "${NEW_SITE}" != "" ]; then echo -en "${idsCL[Cyan]}${NM_HOSTNAMES[${nip}]} (${nip}) ${idsCL[Default]}" @@ -237,7 +237,7 @@ HAPROXY-CONFIG() { nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}" @@ -256,7 +256,7 @@ HAPROXY-CONFIG() { nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}" @@ -274,7 +274,7 @@ HAPROXY-CONFIG() { nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}" @@ -302,7 +302,7 @@ KEEPALIVE-CONFIG() { nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}" @@ -321,7 +321,7 @@ KEEPALIVE-CONFIG() { nip='localhost ' NCMD='' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" fi echo -en "Creating File on ${NM_HOSTNAMES[${nip}]} ($nip)... ${idsCL[Default]}" @@ -588,9 +588,9 @@ BACKUP_OFFSITEPFSENSE(){ WeeklyBackupsToKeep=8 MonthlyBackupsToKeep=3 - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} tar -c -f /cf/conf/conf_backup.tar.gz /cf/conf + ${SSHCMD} root@${NM_OFFSITE_GW_IP} tar -c -f /cf/conf/conf_backup.tar.gz /cf/conf scp root@${NM_OFFSITE_GW_IP}:/cf/conf/conf* ${BAKFOLDER}/${newBAKFOLDER}/ - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NM_OFFSITE_GW_IP} rm -f /cf/conf/conf_backup.tar.gz + ${SSHCMD} root@${NM_OFFSITE_GW_IP} rm -f /cf/conf/conf_backup.tar.gz cd ${BAKFOLDER} if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then @@ -627,7 +627,7 @@ BACKUP_OFFSITEPFSENSE(){ done; if [ $keep == 0 ]; then delFolder="V:\\UniFi-Backups\\Offsite-pfSense_Backups\\${folder##*/}" - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" + ${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" fi fi done @@ -659,10 +659,10 @@ BACKUP_UNIFIUDM(){ WeeklyBackupsToKeep=2 MonthlyBackupsToKeep=1 - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} tar -c -f /volume1/data_backup.tar.gz /data - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} tar -c -f /volume1/root_backup.tar.gz /root + ${SSHCMD} root@${UNIFIUDM_IP} tar -c -f /volume1/data_backup.tar.gz /data + ${SSHCMD} root@${UNIFIUDM_IP} tar -c -f /volume1/root_backup.tar.gz /root scp root@${UNIFIUDM_IP}:/volume1/*_backup.tar.gz ${BAKFOLDER}/${newBAKFOLDER}/ - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFIUDM_IP} rm -f /volume1/*_backup.tar.gz + ${SSHCMD} root@${UNIFIUDM_IP} rm -f /volume1/*_backup.tar.gz cd ${BAKFOLDER} if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then @@ -699,7 +699,7 @@ BACKUP_UNIFIUDM(){ done; if [ $keep == 0 ]; then delFolder="D:\\BackupSync\\UniFi-Backups\\UniFi-UDM_Backups\\${folder##*/}" - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" + ${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" fi fi done @@ -731,12 +731,12 @@ BACKUP_UNIFINVR(){ WeeklyBackupsToKeep=2 MonthlyBackupsToKeep=1 - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/data_backup.tar.gz /data - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/root_backup.tar.gz /root - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} tar -c -f /volume1/unifi_backup.tar.gz /volume1/.srv/unifi-protect/backups /volume1/.srv/unifi-protect/data /volume1/.srv/unifi-protect/dbBackups + ${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/data_backup.tar.gz /data + ${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/root_backup.tar.gz /root + ${SSHCMD} root@${UNIFINVR_IP} tar -c -f /volume1/unifi_backup.tar.gz /volume1/.srv/unifi-protect/backups /volume1/.srv/unifi-protect/data /volume1/.srv/unifi-protect/dbBackups scp root@${UNIFINVR_IP}:/volume1/*_backup.tar.gz ${BAKFOLDER}/${newBAKFOLDER}/ - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${UNIFINVR_IP} rm -f /volume1/*_backup.tar.gz + ${SSHCMD} root@${UNIFINVR_IP} rm -f /volume1/*_backup.tar.gz cd ${BAKFOLDER} if [ "$(date +%d)" == "${DOM}" ] && [ "$(date +%u)" == "${DOW}" ]; then @@ -773,7 +773,7 @@ BACKUP_UNIFINVR(){ done; if [ $keep == 0 ]; then delFolder="D:\\BackupSync\\UniFi-Backups\\UniFi-NVR_Backups\\${folder##*/}" - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" + ${SSHCMD} backup@10.10.1.60 cmd /C rmdir /S /Q "${delFolder}" fi fi done @@ -817,7 +817,7 @@ RUN_COMMAND(){ NCMD='' LH='- localhost' else - NCMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip}" + NCMD="${SSHCMD} root@${nip}" LH='' fi echo -e " ${idsST[Bold]}${idsCL[LightCyan]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]} ${idsCL[LightYellow]}${LH}${idsCL[Default]}" @@ -1103,7 +1103,7 @@ MAINTENANCE_MODE(){ DIVIDER . lightCyan ${DV_LEN} fi PH=${NTYPE}_HOSTS[0] - PH_CMD="ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${!PH}" + PH_CMD="${SSHCMD} root@${!PH}" var2=${NTYPE}_HOSTS[@] for nip in "${!hosts}"; do echo -e " ${idsCL[Green]}${NM_HOSTNAMES[${!PH}]}[${!PH}] <--> ${idsST[Bold]}${NM_HOSTNAMES[${nip}]}[${nip}]${idsST[Reset]}${idsCL[Default]}" @@ -1144,8 +1144,8 @@ ESXI_ISCSI_STORAGE_RESCAN(){ for NIP in "${ESXIHOSTS[@]}"; do if [ $(CHECK_HOST ${NIP}) == true ]; then - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NIP} esxcli storage core adapter rescan --all & - ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${NIP} vmkfstools -V & + ${SSHCMD} root@${NIP} esxcli storage core adapter rescan --all & + ${SSHCMD} root@${NIP} vmkfstools -V & fi done @@ -1173,7 +1173,7 @@ SETUPSSH(){ ssh-copy-id root@${nip} fi - # ssh -q -o ConnectTimeout=3 -o ConnectionAttempts=1 root@${nip} "nmg setupssh_back" + # ${SSHCMD} root@${nip} "nmg setupssh_back" # echo -en "\n${idsCL[LightCyan]}Verifying NodeMgmt ... ${idsCL[Default]}" # NODEUPDATE ${nip} verify "ssh"