408 lines
13 KiB
Bash
Executable File
408 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# MediaManager Scripts
|
|
|
|
action="$1"
|
|
|
|
VERBOSE=true
|
|
|
|
if [ -f /opt/idssys/settings/mediamanager.conf ]; then
|
|
mv /opt/idssys/settings/mediamanager.conf /opt/idssys/mediamanager/settings.conf
|
|
fi
|
|
|
|
|
|
source /opt/idssys/defaults/colors.inc
|
|
source /opt/idssys/defaults/default.inc
|
|
source /opt/idssys/mediamanager/settings.conf
|
|
source /opt/idssys/mediamanager/defaults.inc
|
|
|
|
if [ "${action}" != "check" ] && [ "${action}" != "update" ] && [ "${2}" != "noup" ]; then
|
|
echo ""
|
|
echo -e "${idsCL[LightGreen]} MediaManager ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${idsCL[Default]}"
|
|
DIVIDER . lightGreen
|
|
fi
|
|
|
|
logfile=/opt/idssys/mediamanager/logfile
|
|
|
|
touch $logfile
|
|
|
|
|
|
IFS=,
|
|
SERVICES_CHECK=(${SERVICESCHECK})
|
|
unset IFS
|
|
|
|
MMSTART(){
|
|
[ $VERBOSE = true ] && chktmstart=`date +%s`
|
|
if [ "${1}" = "start" ] || [ "${1}" = "" ]; then
|
|
GOFORSTART=true;
|
|
elif [ "${1}" = "check" ] && [ "$(systemctl is-active openvpn)" != "active" ]; then
|
|
GOFORSTART=true;
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "2nd Failure, will try and restart the VPN" | mail -s "Media Management systems started" ${EMAIL_NOTICE}
|
|
else
|
|
GOFORSTART=false;
|
|
fi
|
|
if [ ${GOFORSTART} = true ]; then
|
|
mmsv='Starting'
|
|
echo
|
|
OUT=$(ip link | grep "$VPN_INTERFACE")
|
|
if [ ${#OUT} = 0 ]; then
|
|
echo -en "${idsCL[LightCyan]}Starting the VPN Tunnel ... ${idsCL[Default]}"
|
|
"${VPN_START[@]}"
|
|
sleep 10s
|
|
else
|
|
echo -en "${idsCL[Green]}The VPN Tunnel is already running ... ${idsCL[Default]}"
|
|
fi
|
|
else
|
|
mmsv='Verifying'
|
|
fi
|
|
|
|
[ $VERBOSE = true ] && echo "$(date) - ${mmsv} VPN..." >> $logfile
|
|
|
|
|
|
OUT=$(ip link | grep "$VPN_INTERFACE")
|
|
if [ ${#OUT} = 0 ]; then
|
|
if [ -f /opt/mm.vpn.fail ]; then
|
|
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
|
|
echo "$(date) - VPN Failure #2 - restarting system" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "2nd Failure, will try and restart the VPN" | mail -s "VPN Failure" ${EMAIL_NOTICE}
|
|
touch /opt/mm.vpn.fail2
|
|
rm -f /opt/mm.vpn.fail
|
|
MMSTOP
|
|
sleep 5m
|
|
"${VPN_START[@]}" &
|
|
sleep 30s
|
|
rm -f /opt/mm.stop
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
elif [ -f /opt/mm.vpn.fail2 ]; then
|
|
echo "$(date) - VPN Failure #3 - Shutting down Media Management systems" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "Final Failure, shutting down system" | mail -s "VPN Failure" ${EMAIL_NOTICE}
|
|
MMSTOP
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
else
|
|
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
|
|
echo "$(date) - VPN Failure #1 - restarting system" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "1st Failure, will try and restart the VPN" | mail -s "VPN Failure" ${EMAIL_NOTICE}
|
|
touch /opt/mm.vpn.fail
|
|
MMSTOP
|
|
sleep 30s
|
|
"${VPN_START[@]}" &
|
|
sleep 30s
|
|
rm -f /opt/mm.stop
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
fi
|
|
|
|
else
|
|
|
|
if [ "$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)" != "$(ip route show | grep default | cut -d' ' -f 3)" ]; then
|
|
[ $VERBOSE = true ] && echo "$(date) - Adding VPN default routes" >> $logfile
|
|
/bin/ip route del default via 10.10.0.1
|
|
/bin/ip route add default via $(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
|
|
fi
|
|
|
|
if [ ${GOFORSTART} = true ]; then
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
fi
|
|
|
|
echo -en "${idsCL[LightCyan]}Verifing default network routes... ${idsCL[Default]}"
|
|
checked=false
|
|
cc=0
|
|
until [ "${checked}" = "" ]; do
|
|
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
|
|
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
|
|
if [ "${zerogw}" = "${defaultgw}" ]; then
|
|
checked=""
|
|
fi
|
|
if [ "${checked}" == "false" ] && [ ${cc} -eq 10 ]; then
|
|
echo -e "${idsCL[Red]}Could not confirm default routes, restarting system!${idsCL[Default]}"
|
|
echo "$(date) - Could not confirm default routes, restarting system" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "Default route failure, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE}
|
|
touch /opt/mm.vpn.fail
|
|
MMSTOP
|
|
sleep 2m
|
|
"${VPN_START[@]}" &
|
|
sleep 1m
|
|
rm -f /opt/mm.stop
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
fi
|
|
((cc=${cc}+1))
|
|
sleep 1s
|
|
done
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
|
|
|
|
echo -en "${idsCL[LightCyan]}Verifing traffic is routing through VPN... ${idsCL[Default]}"
|
|
|
|
scip=""
|
|
mmip=""
|
|
cc=0
|
|
until [ "${scip}" != "" ] && [ "${mmip}" != "" ] && [ "${scip}" != ";; connection timed out; no servers could be reached" ] && [ "${mmip}" != ";; connection timed out; no servers could be reached" ]; do
|
|
scip=$(dig +short sc1.scity.us @9.9.9.9)
|
|
mmip=$(dig +short myip.opendns.com @resolver1.opendns.com)
|
|
if ([ "${scip}" = "" ] || [ "${mmip}" = "" ]) && [ ${cc} -eq 10 ]; then
|
|
echo -e "${idsCL[Red]}Cannot get an WAN address, restarting system!${idsCL[Default]}"
|
|
echo "$(date) - Cannot get an WAN address, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
|
|
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "Cannot get an WAN address, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE}
|
|
touch /opt/mm.vpn.fail
|
|
MMSTOP
|
|
sleep 2m
|
|
"${VPN_START[@]}" &
|
|
sleep 1m
|
|
rm -f /opt/mm.stop
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
fi
|
|
((cc=${cc}+1))
|
|
sleep 1s
|
|
done
|
|
if [ "${scip}" != "${mmip}" ]; then
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
[ $VERBOSE = true ] && echo "$(date) - WAN Address: ${mmip} - GW Address: ${defaultgw}" >> $logfile
|
|
else
|
|
echo -e "${idsCL[Red]}Traffic is not routing through VPN, restarting system!${idsCL[Default]}"
|
|
echo "$(date) - Traffic is not routing through VPN, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
|
|
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "Traffic route failure, will try and restart the VPN" | mail -s "Network Failure" ${EMAIL_NOTICE}
|
|
touch /opt/mm.vpn.fail
|
|
MMSTOP
|
|
sleep 2m
|
|
"${VPN_START[@]}" &
|
|
sleep 1m
|
|
rm -f /opt/mm.stop
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
exit 1
|
|
fi
|
|
|
|
echo -en "${idsCL[LightCyan]}${mmsv} Services... ${idsCL[Default]}"
|
|
for srvc in "${SERVICES_CHECK[@]}"; do
|
|
/bin/systemctl start ${srvc}
|
|
done
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
echo
|
|
if [ "${1}" == "start" ]; then
|
|
#echo -e "${idsCL[LightCyan]}It may take a couple of minutes before all services fully start${idsCL[Default]}"
|
|
MMSTATUS
|
|
fi
|
|
if [ -f /opt/mm.vpn.fail* ]; then
|
|
echo "$(date) - VPN issue has been fixed" >> $logfile
|
|
[ "${EMAIL_NOTICE}" != "" ] && echo "VPN issue has been fixed" | mail -s "Services Restored" ${EMAIL_NOTICE}
|
|
rm -f /opt/mm.vpn.fail*
|
|
fi
|
|
rm -f /opt/mm.stop
|
|
echo -e "${idsCL[Green]}All systems are go!${idsCL[Default]}"
|
|
if [ "${1}" = "start" ] || [ "${1}" = "" ]; then
|
|
echo "$(date) - Media Management systems started" >> $logfile
|
|
fi
|
|
|
|
fi
|
|
echo
|
|
|
|
[ $VERBOSE = true ] && DEBUGLOGSTOP
|
|
|
|
#exit 0
|
|
}
|
|
|
|
MMSTOP(){
|
|
if [ -f /opt/mm.stop ]; then
|
|
echo -e "${idsCL[Yellow]}The system is already stopped, will verify anyway...${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Green]}Stopping System...${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
echo
|
|
touch /opt/mm.stop
|
|
echo -en "${idsCL[LightCyan]}Stopping Services... ${idsCL[Default]}"
|
|
#for srvc in "${SERVICES_CHECK[@]}"; do
|
|
for i in "${!NM_SERVICES[@]}"; do
|
|
srvc=${NM_SERVICES[$i]}
|
|
if [[ ${i} != *"vpn"* ]]; then
|
|
/bin/systemctl stop ${i} &
|
|
fi
|
|
done
|
|
checked=false
|
|
until [ "${checked}" = "" ]; do
|
|
if ! lsof -Pi :5000 -sTCP:LISTEN -t >/dev/null; then #ombi
|
|
if ! lsof -Pi :5555 -sTCP:LISTEN -t >/dev/null; then #radarr
|
|
if ! lsof -Pi :5556 -sTCP:LISTEN -t >/dev/null; then #radarr3d
|
|
if ! lsof -Pi :5656 -sTCP:LISTEN -t >/dev/null; then #sonarr
|
|
if ! lsof -Pi :5858 -sTCP:LISTEN -t >/dev/null; then #qbittorrent
|
|
if ! lsof -Pi :8181 -sTCP:LISTEN -t >/dev/null; then #tautulli
|
|
if ! lsof -Pi :8686 -sTCP:LISTEN -t >/dev/null; then #lidarr
|
|
if ! lsof -Pi :9117 -sTCP:LISTEN -t >/dev/null; then #jackett
|
|
checked=""
|
|
fi #/jackett
|
|
fi #/lidarr
|
|
fi #/tautulli
|
|
fi #/qbittorrent
|
|
fi #/sonarr
|
|
fi #/radarr3d
|
|
fi #/radarr
|
|
fi #/ombi
|
|
done
|
|
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
echo
|
|
echo -en "${idsCL[LightCyan]}Stopping VPN Tunnel... ${idsCL[Default]}"
|
|
"${VPN_STOP[@]}"
|
|
|
|
checked=false
|
|
until [ "${checked}" = "" ]; do
|
|
OUT=$(ip link | grep "$VPN_INTERFACE")
|
|
#OUT=`ip a show $VPN_INTERFACE up` >/dev/null 2>&1
|
|
if [ ${#OUT} = 0 ]; then
|
|
checked=""
|
|
fi
|
|
done
|
|
echo "$(date) - Media Management systems stopped" >> $logfile
|
|
/bin/ip route add default via 10.10.0.1
|
|
[ $VERBOSE = true ] && echo "$(date) - Added default route back in" >> $logfile
|
|
echo -e "${idsCL[Green]} done${idsCL[Default]}"
|
|
|
|
echo
|
|
echo -e "${idsCL[Red]}All services have been stopped${idsCL[Default]}"
|
|
echo
|
|
|
|
#exit 0
|
|
}
|
|
|
|
MMSTATUS(){
|
|
# if [[ ${MMSRV} == *"10.10.1.24"* ]]; then
|
|
# statusArray=("nordvpn")
|
|
# else
|
|
# statusArray=("openvpn")
|
|
# fi
|
|
#statusArray=(nordvpn openvpn)
|
|
statusArray=(openvpn)
|
|
statusArray+=(${SERVICES_CHECK[@]})
|
|
echo
|
|
for srvc in "${statusArray[@]}"; do
|
|
c=0; cw=36 spc=''
|
|
spc1=$((${cw}-${#NM_SERVICES[${srvc}]}))
|
|
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
|
|
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
|
|
|
|
# if [[ ${MMSRV} == *"10.10.1.24"* ]] && [ "${srvc}" = "nordvpn" ]; then
|
|
if [ "${srvc}" = "nordvpn" ]; then
|
|
if [[ "$(nordvpn status)" == *"Status: Disconnected"* ]]; then
|
|
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
DIVIDER . lightBlue
|
|
nordvpn status
|
|
DIVIDER . lightBlue
|
|
echo ""
|
|
fi
|
|
else
|
|
if [ "$(systemctl is-active ${srvc})" != "active" ]; then
|
|
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
|
|
if [[ ${srvc} == *"vpn"* ]]; then
|
|
zerogw=$(netstat -r | grep '0.0.0.0' | head -1 | awk '{ print $2}')
|
|
defaultgw=$(netstat -r | grep 'default' | head -1 | awk '{ print $2}')
|
|
echo -e " |- 0.0.0.0 gateway --> (${zerogw})"
|
|
echo -en " |- DEFAULT gateway --> (${defaultgw}) : "
|
|
if [ "${zerogw}" = "${defaultgw}" ]; then
|
|
echo -e "${idsCL[Green]}Matches 0.0.0.0 route${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}Does not match 0.0.0.0 route${idsCL[Default]}"
|
|
if [ ! -f /opt/mm.stop ]; then
|
|
echo
|
|
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
|
|
MMSTOP
|
|
sleep 30s
|
|
MMSTART
|
|
fi
|
|
fi
|
|
|
|
echo -en " |- Verify traffic route to VPN -> : "
|
|
scip=$(dig +short sc1.scity.us @ns1.scity.us)
|
|
mmip=$(dig +short myip.opendns.com @resolver1.opendns.com)
|
|
if [ "${scip}" != "${mmip}" ]; then
|
|
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
|
|
else
|
|
echo -e "${idsCL[Red]}Not Good${idsCL[Default]}"
|
|
if [ ! -f /opt/mm.stop ]; then
|
|
echo
|
|
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
|
|
MMSTOP
|
|
sleep 30s
|
|
MMSTART
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
done
|
|
echo
|
|
echo
|
|
}
|
|
|
|
if [ ! -f /opt/mm.update.stop ]; then
|
|
if [ -f /opt/mm.stop ]; then
|
|
stoppedsince=`date +%s`-$(stat -c %Y /opt/mm.stop)
|
|
stopmsg="${idsCL[Yellow]}The system has currently been stopped for '$(SHOW_TIME ${stoppedsince})'.\n${idsCL[LightCyan]}(You'll need to run \"mm start\" to start services again)${idsCL[Default]}"
|
|
fi
|
|
case $action in
|
|
start | check)
|
|
if [ -f /opt/mm.stop ] && [ "${action}" = "check" ]; then
|
|
echo -e "${stopmsg}"
|
|
# [ $VERBOSE = true ] && echo "$(date) - Not checking, system has been stopped" >> $logfile
|
|
echo
|
|
exit 1
|
|
fi
|
|
touch /opt/mm.stop
|
|
MMSTART ${action};;
|
|
|
|
stop)
|
|
MMSTOP;;
|
|
|
|
restart)
|
|
if [ -f /opt/mm.stop ]; then
|
|
echo -e "${stopmsg}"
|
|
echo
|
|
exit 1
|
|
fi
|
|
MMSTOP
|
|
sleep 10s
|
|
MMSTART;;
|
|
|
|
status)
|
|
if [ -f /opt/mm.stop ]; then
|
|
echo -e "${stopmsg}"
|
|
echo
|
|
fi
|
|
MMSTATUS;;
|
|
|
|
update);;
|
|
|
|
*)
|
|
echo -e "Usage: ${idsCL[LightCyan]}mm${idsCL[Default]} {"
|
|
echo -e " ${idsCL[Yellow]}start${idsCL[Default]} --> Start services"
|
|
echo -e " ${idsCL[Yellow]}stop${idsCL[Default]} --> Stop services"
|
|
echo -e " ${idsCL[Yellow]}restart${idsCL[Default]} --> Restart services"
|
|
echo -e " ${idsCL[Yellow]}check${idsCL[Default]} --> Check for VPN and start services if needed"
|
|
echo -e " ${idsCL[Yellow]}status${idsCL[Default]} --> Check service status"
|
|
echo -e "}"
|
|
echo ""
|
|
echo ""
|
|
exit 0;;
|
|
esac
|
|
|
|
else
|
|
echo -e "${idsCL[Yellow]}Part of the system is currently updating, please try again shortly.\nUpdates usually only take 1-3mins.${idsCL[Default]}"
|
|
echo
|
|
fi
|
|
|
|
|
|
exit 0
|
|
|
|
|
|
|