#!/usr/bin/env bash # MediaManager Scripts action="$1" VERBOSE=true if [ -f /opt/idssys/settings/mediamanager.conf ]; then mv /opt/idssys/settings/mediamanager.conf /opt/idssys/mediamanager/settings.conf fi source /opt/idssys/defaults/colors.inc source /opt/idssys/defaults/default.inc source /opt/idssys/mediamanager/settings.conf source /opt/idssys/mediamanager/defaults.inc if [ "${action}" != "check" ] && [ "${action}" != "update" ] && [ "${2}" != "noup" ]; then echo "" echo -e "${idsCL[LightGreen]} MediaManager ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}" DIVIDER . lightGreen fi logfile=/opt/idssys/mediamanager/logfile touch $logfile IFS=, SERVICES_CHECK=(${SERVICESCHECK}) unset IFS MMSTART(){ [ $VERBOSE = true ] && chktmstart=`date +%s` if [ "${1}" = "start" ] || [ "${1}" = "" ]; then GOFORSTART=true; elif [ "${1}" = "check" ] && [ "$(systemctl is-active openvpn)" != "active" ] && [ ! -f /opt/mm.vpn* ]; then GOFORSTART=true; [ "${EMAIL_NOTICE}" != "" ] && echo "" | mail -s "Starting Media Management Systems" ${EMAIL_NOTICE} echo "$(date) - Starting Media Management system - (openvpn not running)" >> $logfile echo -e "${idsCL[LightCyan]}Fresh startup${idsCL[Default]}" else GOFORSTART=false; fi if [ ${GOFORSTART} = true ]; then mmsv='Starting' echo OUT=$(ip link | grep "$VPN_INTERFACE") if [ ${#OUT} = 0 ]; then echo -en "${idsCL[LightCyan]}Starting the VPN Tunnel ... ${idsCL[Default]}" "${VPN_START[@]}" sleep 10s else echo -en "${idsCL[Green]}The VPN Tunnel is already running ... ${idsCL[Default]}" fi else mmsv='Verifying' fi [ $VERBOSE = true ] && echo "$(date) - ${mmsv} VPN..." >> $logfile OUT=$(ip link | grep "$VPN_INTERFACE") if [ ${#OUT} = 0 ]; then if [ -f /opt/mm.vpn.fail ]; then echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}" echo "$(date) - VPN Failure #2 - restarting system" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "2nd Failure, will try and restart the VPN" | mail -s "VPN Failure" ${EMAIL_NOTICE} touch /opt/mm.vpn.fail2 rm -f /opt/mm.vpn.fail MMSTOP sleep 5m "${VPN_START[@]}" & sleep 30s rm -f /opt/mm.stop [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 elif [ -f /opt/mm.vpn.fail2 ]; then echo "$(date) - VPN Failure #3 - Shutting down Media Management systems" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "Final Failure, shutting down system" | mail -s "VPN Failure" ${EMAIL_NOTICE} MMSTOP [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 else echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}" echo "$(date) - VPN Failure #1 - restarting system" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "1st Failure, will try and restart the VPN" | mail -s "VPN Failure" ${EMAIL_NOTICE} touch /opt/mm.vpn.fail MMSTOP sleep 30s "${VPN_START[@]}" & sleep 30s rm -f /opt/mm.stop [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 fi else zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3) defaultgw=$(ip route show | grep default | cut -d' ' -f 3) if [ ${GOFORSTART} = true ]; then echo -e "${idsCL[Green]} done${idsCL[Default]}" fi if [ "${zerogw}" != "${defaultgw}" ]; then [ $VERBOSE = true ] && echo "$(date) - Adding VPN default routes" >> $logfile echo -en "${idsCL[LightCyan]}Adding VPN default routes... ${idsCL[Default]}" /bin/ip route del default via 10.10.0.1 /bin/ip route add default via ${zerogw} echo -e "${idsCL[Green]} done${idsCL[Default]}" fi echo -en "${idsCL[LightCyan]}Verifing default network routes... ${idsCL[Default]}" checked=false cc=0 until [ "${checked}" = "" ]; do if [ "${zerogw}" = "${defaultgw}" ]; then checked="" fi if [ "${checked}" == "false" ] && [ ${cc} -eq 10 ]; then echo -e "${idsCL[Red]}Could not confirm default routes, restarting system!${idsCL[Default]}" echo "$(date) - Could not confirm default routes, restarting system" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "Default route failure, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE} touch /opt/mm.vpn.fail MMSTOP sleep 2m "${VPN_START[@]}" & sleep 1m rm -f /opt/mm.stop [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 fi ((cc=${cc}+1)) sleep 1s zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3) defaultgw=$(ip route show | grep default | cut -d' ' -f 3) done echo -e "${idsCL[Green]} done${idsCL[Default]}" echo -en "${idsCL[LightCyan]}Verifing traffic is routing through VPN... ${idsCL[Default]}" scip="" mmip="" cc=0 until [ "${scip}" != "" ] && [ "${mmip}" != "" ] && [ "${scip}" != ";; connection timed out; no servers could be reached" ] && [ "${mmip}" != ";; connection timed out; no servers could be reached" ]; do scip=$(dig +short sc1.scity.us @9.9.9.9) mmip=$(dig +short myip.opendns.com @resolver1.opendns.com) if ([ "${scip}" = "" ] || [ "${mmip}" = "" ]) && [ ${cc} -eq 10 ]; then echo -e "${idsCL[Red]}Cannot get an WAN address, restarting system!${idsCL[Default]}" echo "$(date) - Cannot get an WAN address, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile [ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "Cannot get an WAN address, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE} touch /opt/mm.vpn.fail MMSTOP sleep 2m "${VPN_START[@]}" & sleep 1m rm -f /opt/mm.stop [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 fi ((cc=${cc}+1)) sleep 1s done if [ "${scip}" != "${mmip}" ]; then echo -e "${idsCL[Green]} done${idsCL[Default]}" [ $VERBOSE = true ] && echo "$(date) - WAN Address: ${mmip} - GW Address: ${defaultgw}" >> $logfile else echo -e "${idsCL[Red]}Traffic is not routing through VPN, restarting system!${idsCL[Default]}" echo "$(date) - Traffic is not routing through VPN, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile [ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "Traffic route failure, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE} touch /opt/mm.vpn.fail MMSTOP sleep 2m "${VPN_START[@]}" & sleep 1m rm -f /opt/mm.stop [ $VERBOSE = true ] && DEBUGLOGSTOP exit 1 fi echo -en "${idsCL[LightCyan]}${mmsv} Services... ${idsCL[Default]}" for srvc in "${SERVICES_CHECK[@]}"; do /bin/systemctl start ${srvc} done echo -e "${idsCL[Green]} done${idsCL[Default]}" echo if [ "${1}" == "start" ]; then #echo -e "${idsCL[LightCyan]}It may take a couple of minutes before all services fully start${idsCL[Default]}" MMSTATUS fi if [ -f /opt/mm.vpn.fail* ]; then echo "$(date) - VPN issue has been fixed" >> $logfile [ "${EMAIL_NOTICE}" != "" ] && echo "VPN issue has been fixed" | mail -s "Services Restored" ${EMAIL_NOTICE} rm -f /opt/mm.vpn.fail* fi rm -f /opt/mm.stop echo -e "${idsCL[Green]}All systems are go!${idsCL[Default]}" if [ "${1}" = "start" ] || [ "${1}" = "" ]; then echo "$(date) - Media Management systems started" >> $logfile fi fi echo [ $VERBOSE = true ] && DEBUGLOGSTOP #exit 0 } MMSTOP(){ if [ -f /opt/mm.stop ]; then echo -e "${idsCL[Yellow]}The system is already stopped, will verify anyway...${idsCL[Default]}" else echo -e "${idsCL[Green]}Stopping System...${idsCL[Default]}" echo fi echo touch /opt/mm.stop echo -en "${idsCL[LightCyan]}Stopping Services... ${idsCL[Default]}" #for srvc in "${SERVICES_CHECK[@]}"; do for i in "${!NM_SERVICES[@]}"; do srvc=${NM_SERVICES[$i]} if [[ ${i} != *"vpn"* ]]; then /bin/systemctl stop ${i} & fi done checked=false until [ "${checked}" = "" ]; do if ! lsof -Pi :5000 -sTCP:LISTEN -t >/dev/null; then #ombi if ! lsof -Pi :5555 -sTCP:LISTEN -t >/dev/null; then #radarr if ! lsof -Pi :5556 -sTCP:LISTEN -t >/dev/null; then #radarr3d if ! lsof -Pi :5656 -sTCP:LISTEN -t >/dev/null; then #sonarr if ! lsof -Pi :5858 -sTCP:LISTEN -t >/dev/null; then #qbittorrent if ! lsof -Pi :8181 -sTCP:LISTEN -t >/dev/null; then #tautulli if ! lsof -Pi :8686 -sTCP:LISTEN -t >/dev/null; then #lidarr if ! lsof -Pi :9117 -sTCP:LISTEN -t >/dev/null; then #jackett checked="" fi #/jackett fi #/lidarr fi #/tautulli fi #/qbittorrent fi #/sonarr fi #/radarr3d fi #/radarr fi #/ombi done echo -e "${idsCL[Green]} done${idsCL[Default]}" echo echo -en "${idsCL[LightCyan]}Stopping VPN Tunnel... ${idsCL[Default]}" "${VPN_STOP[@]}" checked=false until [ "${checked}" = "" ]; do OUT=$(ip link | grep "$VPN_INTERFACE") #OUT=`ip a show $VPN_INTERFACE up` >/dev/null 2>&1 if [ ${#OUT} = 0 ]; then checked="" fi done echo "$(date) - Media Management systems stopped" >> $logfile /bin/ip route add default via 10.10.0.1 [ $VERBOSE = true ] && echo "$(date) - Added default route back in" >> $logfile echo -e "${idsCL[Green]} done${idsCL[Default]}" echo echo -e "${idsCL[Red]}All services have been stopped${idsCL[Default]}" echo #exit 0 } MMSTATUS(){ # if [[ ${MMSRV} == *"10.10.1.24"* ]]; then # statusArray=("nordvpn") # else # statusArray=("openvpn") # fi #statusArray=(nordvpn openvpn) statusArray=(openvpn) statusArray+=(${SERVICES_CHECK[@]}) echo for srvc in "${statusArray[@]}"; do c=0; cw=36 spc='' spc1=$((${cw}-${#NM_SERVICES[${srvc}]})) until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done echo -en " ${NM_SERVICES[${srvc}]}$spc: " # if [[ ${MMSRV} == *"10.10.1.24"* ]] && [ "${srvc}" = "nordvpn" ]; then if [ "${srvc}" = "nordvpn" ]; then if [[ "$(nordvpn status)" == *"Status: Disconnected"* ]]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" else echo -e "${idsCL[Green]}Running${idsCL[Default]}" DIVIDER . lightBlue nordvpn status DIVIDER . lightBlue echo "" fi else if [ "$(systemctl is-active ${srvc})" != "active" ]; then echo -e "${idsCL[Red]}Not Running${idsCL[Default]}" else echo -e "${idsCL[Green]}Running${idsCL[Default]}" if [[ ${srvc} == *"vpn"* ]]; then zerogw=$(netstat -r | grep '0.0.0.0' | head -1 | awk '{ print $2}') defaultgw=$(netstat -r | grep 'default' | head -1 | awk '{ print $2}') echo -e " |- 0.0.0.0 gateway --> (${zerogw})" echo -en " |- DEFAULT gateway --> (${defaultgw}) : " if [ "${zerogw}" = "${defaultgw}" ]; then echo -e "${idsCL[Green]}Matches 0.0.0.0 route${idsCL[Default]}" else echo -e "${idsCL[Red]}Does not match 0.0.0.0 route${idsCL[Default]}" if [ ! -f /opt/mm.stop ]; then echo echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}" MMSTOP sleep 30s MMSTART fi fi echo -en " |- Verify traffic route to VPN -> : " scip=$(dig +short sc1.scity.us @ns1.scity.us) mmip=$(dig +short myip.opendns.com @resolver1.opendns.com) if [ "${scip}" != "${mmip}" ]; then echo -e "${idsCL[Green]}Good${idsCL[Default]}" else echo -e "${idsCL[Red]}Not Good${idsCL[Default]}" if [ ! -f /opt/mm.stop ]; then echo echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}" MMSTOP sleep 30s MMSTART fi fi echo fi fi fi done echo echo } if [ ! -f /opt/mm.update.stop ]; then if [ -f /opt/mm.stop ]; then stoppedsince=`date +%s`-$(stat -c %Y /opt/mm.stop) stopmsg="${idsCL[Yellow]}The system has currently been stopped for '$(SHOW_TIME ${stoppedsince})'.\n${idsCL[LightCyan]}(You'll need to run \"mm start\" to start services again)${idsCL[Default]}" fi case $action in start | check) if [ -f /opt/mm.stop ] && [ "${action}" = "check" ]; then echo -e "${stopmsg}" # [ $VERBOSE = true ] && echo "$(date) - Not checking, system has been stopped" >> $logfile echo exit 1 fi touch /opt/mm.stop MMSTART ${action};; stop) MMSTOP;; restart) if [ -f /opt/mm.stop ]; then echo -e "${stopmsg}" echo exit 1 fi MMSTOP sleep 10s MMSTART;; status) if [ -f /opt/mm.stop ]; then echo -e "${stopmsg}" echo fi MMSTATUS;; update);; *) echo -e "Usage: ${idsCL[LightCyan]}mm${idsCL[Default]} {" echo -e " ${idsCL[Yellow]}start${idsCL[Default]} --> Start services" echo -e " ${idsCL[Yellow]}stop${idsCL[Default]} --> Stop services" echo -e " ${idsCL[Yellow]}restart${idsCL[Default]} --> Restart services" echo -e " ${idsCL[Yellow]}check${idsCL[Default]} --> Check for VPN and start services if needed" echo -e " ${idsCL[Yellow]}status${idsCL[Default]} --> Check service status" echo -e "}" echo "" echo "" exit 0;; esac else echo -e "${idsCL[Yellow]}Part of the system is currently updating, please try again shortly.\nUpdates usually only take 1-3mins.${idsCL[Default]}" echo fi exit 0