Files
mediamanager/mm-scripts.sh
2023-03-04 23:17:05 -06:00

553 lines
18 KiB
Bash
Executable File

#!/usr/bin/env bash
# MediaManager Scripts
action="$1"
VERBOSE=true
if [ -f /opt/idssys/settings/mediamanager.conf ]; then
mv /opt/idssys/settings/mediamanager.conf ${MMFOLDER}/settings.conf
fi
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/mediamanager/settings.conf
source /opt/idssys/mediamanager/defaults.inc
if [ "${action}" != "check" ] && [ "${action}" != "update" ] && [ "${2}" != "noup" ]; then
if [ -f ${MMFOLDER}/update.available ]; then
dispup="${idsCL[LightGreen]} - Update Available! (run '${idsCL[LightCyan]}mm update${idsCL[LightGreen]}')${idsCL[Default]}"
else
dispup=""
fi
echo
echo -e "${idsCL[LightGreen]} MediaManager ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${dispup}${idsCL[Default]}"
DIVIDER . lightGreen
fi
logfile=${MMFOLDER}/logfile
touch $logfile
IFS=,
SERVICES_CHECK=(${SERVICESCHECK})
unset IFS
MMSTART(){
[ $VERBOSE = true ] && chktmstart=`date +%s`
if [ "${1}" = "start" ] || [ "${1}" = "restart" ] || [ "${1}" = "" ]; then
GOFORSTART=true;
elif [ "${1}" = "check" ] && [ "$(ip link | grep "${VPN_INTERFACE}")" = "" ] && [ ! -f ${MMFOLDER}/mm.vpn* ]; then
GOFORSTART=true;
SENDNOTICE "Starting Media Management Systems" ""
echo "$(date) - Starting Media Management system - (fresh start)" >> $logfile
echo -e "${idsCL[LightCyan]}Fresh startup${idsCL[Default]}"
elif [ "${1}" = "check" ] && [ "$(ip link | grep "${VPN_INTERFACE}")" = "" ] && [ -f ${MMFOLDER}/mm.vpn* ]; then
GOFORSTART=true;
echo "$(date) - Attempting to restart systems after VPN failure..." >> $logfile
echo -e "${idsCL[LightCyan]}Attempting to restart systems after VPN failure...${idsCL[Default]}"
else
GOFORSTART=false;
fi
if [ ${GOFORSTART} = true ]; then
mmsv='Starting'
echo
OUT=$(ip link | grep "${VPN_INTERFACE}")
if [ ${#OUT} = 0 ]; then
echo -en "${idsCL[LightCyan]}Setting up the VPN Tunnel config :${idsCL[Default]}"
privkey=$(wg genkey)
reqpublickey=$(echo "${privkey}" | wg pubkey)
reqpublickey="$(php -r 'echo rawurlencode($argv[1]);' -- "$reqpublickey")"
JSON=$(curl https://${VPN_SERVER}:1443/api/v1/setup?public-key=${reqpublickey} --user "${VPN_USER}:${VPN_PASS}" -sk)
publickey=`echo ${JSON} | jq -r '.server_public_key'`
vpnaddress=`echo ${JSON} | jq -r '.vpn_server_address'`
vpnport=`echo ${JSON} | jq -r '.vpn_server_port'`
vpnip=`echo ${JSON} | jq -r '.client_ipv4'`
rm -f /etc/wireguard/${VPN_INTERFACE}.conf
echo -e "[Interface]
PrivateKey = ${privkey}
ListenPort = 55009
MTU = 1390
DNS = 1.1.1.1
Address = ${vpnip}
[Peer]
PublicKey = ${publickey}
AllowedIPs = 0.0.0.0/0
Endpoint = ${vpnaddress}:${vpnport}
PersistentKeepalive = 25" >> /etc/wireguard/${VPN_INTERFACE}.conf
rm -f ${MMFOLDER}/vpninfo.*
echo -e "$publickey" >> ${MMFOLDER}/vpninfo.publickey
echo -e "$vpnaddress" >> ${MMFOLDER}/vpninfo.address
echo -e "${idsCL[Green]} done${idsCL[Default]}"
echo -en "${idsCL[LightCyan]}Starting the VPN Tunnel :${idsCL[Default]}"
"${VPN_START[@]}" >/dev/null 2>&1
sleep 5s
else
echo -en "${idsCL[Green]}The VPN Tunnel is already running :${idsCL[Default]}"
fi
else
mmsv='Verifying'
fi
[ $VERBOSE = true ] && echo "$(date) - ${mmsv} VPN..." >> $logfile
OUT=$(ip link | grep "$VPN_INTERFACE")
if [ ${#OUT} = 0 ]; then
if [ -f ${MMFOLDER}/mm.vpn.fail ]; then
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
echo "$(date) - VPN Failure #2 - restarting system" >> $logfile
SENDNOTICE "VPN Failure" "2nd Failure, will try and restart the VPN"
touch ${MMFOLDER}/mm.vpn.fail2
rm -f ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 5m
"${VPN_START[@]}" >/dev/null 2>&1 &
sleep 30s
rm -f ${MMFOLDER}/mm.stop*
rm -f ${MMFOLDER}/mm.checking
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
elif [ -f ${MMFOLDER}/mm.vpn.fail2 ]; then
echo "$(date) - VPN Failure #3 - Shutting down Media Management systems" >> $logfile
SENDNOTICE "VPN Failure" "Final Failure, shutting down system"
MMSTOP
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
else
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
echo "$(date) - VPN Failure #1 - restarting system" >> $logfile
SENDNOTICE "VPN Failure" "1st Failure, will try and restart the VPN"
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 30s
"${VPN_START[@]}" >/dev/null 2>&1 &
sleep 30s
rm -f ${MMFOLDER}/mm.stop*
rm -f ${MMFOLDER}/mm.checking
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
else
if [ ${GOFORSTART} = true ]; then
echo -e "${idsCL[Green]} done${idsCL[Default]}"
fi
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
# if [ "${zerogw}" != "${defaultgw}" ]; then
if [ "${defaultgw}" != "${VPN_INTERFACE}" ]; then
echo "$(date) - Adding VPN default routes" >> $logfile
echo -en "${idsCL[LightCyan]}Adding VPN default routes :${idsCL[Default]}"
/bin/ip route del default
/bin/ip route add default dev ${VPN_INTERFACE}
/bin/ip route add 0.0.0.0/1 dev ${VPN_INTERFACE}
/bin/ip route add 128.0.0.0/1 dev ${VPN_INTERFACE}
/bin/ip route add $(cat ${MMFOLDER}/vpninfo.address)/32 via 10.10.0.1 dev eth0
echo -e "${idsCL[Green]} done${idsCL[Default]}"
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
fi
echo -en "${idsCL[LightCyan]}Verifing default network routes :${idsCL[Default]}"
checked=false
cc=0
until [ "${checked}" = "" ]; do
if [ "${zerogw}" = "${defaultgw}" ]; then
checked=""
fi
if [ "${checked}" == "false" ] && [ ${cc} -eq 10 ]; then
echo -e "${idsCL[Red]}Could not confirm default routes, restarting system!${idsCL[Default]}"
echo "$(date) - Could not confirm default routes, restarting system" >> $logfile
SENDNOTICE "Network Failure" "Default route failure, will try and restart the VPN"
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 2m
"${VPN_START[@]}" >/dev/null 2>&1 &
sleep 1m
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
((cc=${cc}+1))
sleep 1s
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
done
echo -e "${idsCL[Green]} done${idsCL[Default]}"
echo -en "${idsCL[LightCyan]}Verifing traffic routing through VPN :${idsCL[Default]}"
scip=""
mmip=""
cc=0
until [ "${scip}" != "" ] && [ "${mmip}" != "" ] && [ "${scip}" != ";; connection timed out; no servers could be reached" ] && [ "${mmip}" != ";; connection timed out; no servers could be reached" ]; do
scip=$(dig +short sc1.scity.us @9.9.9.9)
mmip=$(dig +short myip.opendns.com @resolver1.opendns.com)
if ([ "${scip}" = "" ] || [ "${mmip}" = "" ]) && [ ${cc} -eq 10 ]; then
echo -e "${idsCL[Red]}Cannot get an WAN address, restarting system!${idsCL[Default]}"
echo "$(date) - Cannot get an WAN address, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
SENDNOTICE "Network Failure" "Cannot get an WAN address, will try and restart the VPN"
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 2m
"${VPN_START[@]}" >/dev/null 2>&1 &
sleep 1m
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
((cc=${cc}+1))
sleep 1s
done
if [ "${scip}" != "${mmip}" ]; then
echo -e "${idsCL[Green]} done${idsCL[Default]}"
[ $VERBOSE = true ] && echo "$(date) - WAN Address: ${mmip} - GW Address: ${defaultgw}" >> $logfile
else
echo -e "${idsCL[Red]}Traffic is not routing through VPN, restarting system!${idsCL[Default]}"
echo "$(date) - Traffic is not routing through VPN, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
SENDNOTICE "Network Failure" "Traffic route failure, will try and restart the VPN"
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 2m
"${VPN_START[@]}" >/dev/null 2>&1 &
sleep 1m
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
### CHECK FOR PLEX MOUNT
files=$(shopt -s nullglob dotglob; echo ${PLEX_MOUNT}/*)
if (( ! ${#files} )); then
echo "$(date) - Plex Media Not Mounted, will try and fix" >> $logfile
umount ${PLEX_MOUNT}
sleep 2s
mount -a
sleep 2s
files=$(shopt -s nullglob dotglob; echo ${PLEX_MOUNT}/*)
if (( ! ${#files} )); then
if [ -f ${MMFOLDER}/mm.plexmount.fail ]; then
fd=$(( `date +%s` - `stat -L --format %Y ${MMFOLDER}/mm.plexmount.fail` ))
if [ ${fd} -gt 3600 ]; then
PMFS=true
rm -f ${MMFOLDER}/mm.plexmount.fail
else
PMFS=false
fi
else
PMFS=true
fi
if [ "${EMAIL_NOTICE}" != "" ] && [ ${PMFS} = true ]; then
SENDNOTICE "Plex Media Not Mounted" "Could not mount the Plex Media Folder"
touch ${MMFOLDER}/mm.plexmount.fail
echo "$(date) - Plex Media could not be mounted" >> $logfile
fi
else
echo "$(date) - Plex Media now mounted, stopping services to restart them" >> $logfile
for srvc in "${SERVICES_CHECK[@]}"; do
/bin/systemctl stop ${srvc}
done
fi
else
rm -f ${MMFOLDER}/mm.plexmount.fail
fi
[ "${1}" == "start" ] && rm -f ${MMFOLDER}/mm.noservices
if [ "${2}" != "vpn" ] && [ ! -f ${MMFOLDER}/mm.noservices ]; then
echo -en "${idsCL[LightCyan]}${mmsv} Services :${idsCL[Default]}"
for srvc in "${SERVICES_CHECK[@]}"; do
/bin/systemctl start ${srvc}
done
echo -e "${idsCL[Green]} done${idsCL[Default]}"
else
echo -e "${idsCL[LightYellow]}Skipping ${mmsv} Services ${idsCL[Default]}"
touch ${MMFOLDER}/mm.noservices
fi
echo
if [ "${1}" == "start" ] || [ "${1}" == "restart" ]; then
#echo -e "${idsCL[LightCyan]}It may take a couple of minutes before all services fully start${idsCL[Default]}"
MMSTATUS
fi
if [ -f ${MMFOLDER}/mm.vpn.fail* ]; then
echo "$(date) - VPN issue has been fixed" >> $logfile
SENDNOTICE "Services Restored" "VPN issue has been fixed"
rm -f ${MMFOLDER}/mm.vpn.fail*
fi
rm -f ${MMFOLDER}/mm.stop*
echo -e "${idsCL[Green]}All systems are go!${idsCL[Default]}"
if [ "${1}" = "start" ] || [ "${1}" = "" ]; then
echo "$(date) - Media Management systems started" >> $logfile
fi
fi
echo
[ $VERBOSE = true ] && DEBUGLOGSTOP
#exit 0
}
MMSTOP(){
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${idsCL[Yellow]}The system is already stopped, will verify anyway...${idsCL[Default]}"
else
echo -e "${idsCL[Green]}Stopping System...${idsCL[Default]}"
echo
fi
echo
touch ${MMFOLDER}/mm.stop
echo -en "${idsCL[LightCyan]}Stopping Services... ${idsCL[Default]}"
#for srvc in "${SERVICES_CHECK[@]}"; do
for i in "${!NM_SERVICES[@]}"; do
srvc=${NM_SERVICES[$i]}
if [[ ${i} != *"vpn"* ]] && [[ ${i} != *"wireguard"* ]]; then
/bin/systemctl stop ${i} &
fi
done
checked=false
until [ "${checked}" = "" ]; do
if ! lsof -Pi :5000 -sTCP:LISTEN -t >/dev/null; then #ombi
if ! lsof -Pi :5555 -sTCP:LISTEN -t >/dev/null; then #radarr
if ! lsof -Pi :5556 -sTCP:LISTEN -t >/dev/null; then #radarr3d
if ! lsof -Pi :5656 -sTCP:LISTEN -t >/dev/null; then #sonarr
if ! lsof -Pi :5858 -sTCP:LISTEN -t >/dev/null; then #qbittorrent
if ! lsof -Pi :8181 -sTCP:LISTEN -t >/dev/null; then #tautulli
if ! lsof -Pi :8686 -sTCP:LISTEN -t >/dev/null; then #lidarr
if ! lsof -Pi :9117 -sTCP:LISTEN -t >/dev/null; then #jackett
checked=""
fi #/jackett
fi #/lidarr
fi #/tautulli
fi #/qbittorrent
fi #/sonarr
fi #/radarr3d
fi #/radarr
fi #/ombi
done
echo -e "${idsCL[Green]} done${idsCL[Default]}"
echo
echo -en "${idsCL[LightCyan]}Stopping VPN Tunnel... ${idsCL[Default]}"
"${VPN_STOP[@]}" >/dev/null 2>&1
checked=false
until [ "${checked}" = "" ]; do
OUT=$(ip link | grep "$VPN_INTERFACE")
#OUT=`ip a show $VPN_INTERFACE up` >/dev/null 2>&1
if [ ${#OUT} = 0 ]; then
checked=""
fi
done
[ "$(ip route show | grep default | cut -d' ' -f 3)" != "10.10.0.1" ] && /bin/ip route add default via 10.10.0.1 && ip route del $(cat ${MMFOLDER}/vpninfo.address)
rm -f ${MMFOLDER}/vpninfo.*
rm -f ${MMFOLDER}/noservices*
[ $VERBOSE = true ] && echo "$(date) - Added default route back in" >> $logfile
echo -e "${idsCL[Green]} done${idsCL[Default]}"
echo "$(date) - Media Management systems stopped" >> $logfile
echo
echo -e "${idsCL[Red]}All services have been stopped${idsCL[Default]}"
echo
#exit 0
}
MMSTATUS(){
statusArray=(wireguard)
statusArray+=(${SERVICES_CHECK[@]})
echo
for srvc in "${statusArray[@]}"; do
c=0; cw=37 spc=''
spc1=$((${cw}-${#NM_SERVICES[${srvc}]}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en " ${NM_SERVICES[${srvc}]}$spc: "
if [ "$(systemctl is-active ${srvc})" != "active" ] && [ "${srvc}" != "wireguard" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
elif [ "${srvc}" == "wireguard" ] && [ "$(ip link | grep $VPN_INTERFACE)" == "" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
else
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
if [[ ${srvc} == *"vpn"* ]] || [[ ${srvc} == *"wireguard"* ]]; then
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
echo -e " |- 0.0.0.0 gateway : ${idsCL[Green]}$zerogw${idsCL[Default]}"
echo -en " |- DEFAULT gateway : "
if [ "${zerogw}" == "${defaultgw}" ] && [ "${zerogw}" == "${VPN_INTERFACE}" ]; then
echo -e "${idsCL[Green]}$defaultgw${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not correct${idsCL[Default]}"
if [ ! -f ${MMFOLDER}/mm.stop ]; then
echo
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 30secs... ${idsCL[Default]}"
sleep 30s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
MMSTART
echo -e " ${NM_SERVICES[wireguard]} : ${idsCL[Green]}Running${idsCL[Default]}"
zerogw=$(ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(ip route show | grep default | cut -d' ' -f 3)
echo -e " |- 0.0.0.0 gateway : ${idsCL[Green]}$zerogw${idsCL[Default]}"
echo -en " |- DEFAULT gateway : "
if [ "${zerogw}" == "${defaultgw}" ] && [ "${zerogw}" == "${VPN_INTERFACE}" ]; then
echo -e "${idsCL[Green]}$defaultgw${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not correct${idsCL[Default]}"
echo
if [ ! -f ${MMFOLDER}/mm.checking ]; then
echo -e "${idsCL[LightCyan]}Shutting down system... fix yo shit${idsCL[Default]}"
MMSTOP
fi
fi
fi
fi
echo -en " |- Verify routing through VPN : "
scip=$(dig +short sc1.scity.us @9.9.9.9)
mmip=$(dig +short myip.opendns.com @resolver1.opendns.com)
if [ "${scip}" != "${mmip}" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not Good${idsCL[Default]}"
if [ ! -f ${MMFOLDER}/mm.stop ] && [ ! -f ${MMFOLDER}/mm.checking ]; then
echo
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 30secs... ${idsCL[Default]}"
sleep 30s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
MMSTART
fi
fi
echo
fi
fi
done
echo
echo
}
if [ ! -f ${MMFOLDER}/mm.update.stop ]; then
if [ -f ${MMFOLDER}/mm.stop ]; then
stoppedsince=`date +%s`-$(stat -c %Y ${MMFOLDER}/mm.stop)
stopmsg="${idsCL[Yellow]}The system has currently been stopped for '$(SHOW_TIME ${stoppedsince})'.\n${idsCL[LightCyan]}(You'll need to run \"mm start\" to start services again)${idsCL[Default]}"
fi
case $action in
start | check)
if [ -f ${MMFOLDER}/mm.stop ] && [ "${action}" = "check" ]; then
if [ -f ${MMFOLDER}/mm.stop.checking ]; then
checkstopped=`expr $(date +%s) - $(stat -c %Y ${MMFOLDER}/mm.stop.checking)`
if [ $checkstopped -gt 1800 ]; then
rm -f ${MMFOLDER}/mm.stop*
getStuckMM=$(pgrep -f '/mm')
for i in $getStuckMM; do
/bin/kill $i
done
echo "$(date) - Bash script stuck in stop mode after doing a check" >> $logfile
SENDNOTICE "MM Script Error" "Bash script stuck in stop mode after doing a check"
else
exit 1
fi
else
echo -e "${stopmsg}"
# [ $VERBOSE = true ] && echo "$(date) - Not checking, system has been stopped" >> $logfile
echo
exit 1
fi
elif [ "${VPN_USER}" == "" ] || [ "${VPN_PASS}" == "" ] || [ "${VPN_SERVER}" == "" ]; then
echo "$(date) - VPN User Info Missing" >> $logfile
echo -e "${idsCL[Red]}VPN User Info Missing${idsCL[Default]}"
echo
exit 1
fi
touch ${MMFOLDER}/mm.stop
[ "${action}" = "check" ] && touch ${MMFOLDER}/mm.stop.checking
MMSTART ${action} ${2};;
stop)
MMSTOP;;
restart)
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${stopmsg}"
echo
exit 1
fi
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 10secs... ${idsCL[Default]}"
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
MMSTART ${action};;
status)
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${stopmsg}"
echo
fi
MMSTATUS;;
update);;
clear) echo -e "${idsCL[LightCyan]}Clearing all mm.* files ${idsCL[Default]}"
echo;;
*)
echo -e " Usage: ${idsCL[LightCyan]}mm {option}${idsCL[Default]}"
echo
echo -e " ${idsCL[Yellow]}start${idsCL[Default]} --> Start services"
echo -e " ${idsCL[Yellow]}stop${idsCL[Default]} --> Stop services"
echo -e " ${idsCL[Yellow]}restart${idsCL[Default]} --> Restart services"
echo -e " ${idsCL[Yellow]}status${idsCL[Default]} --> View VPN and service status"
echo
echo -e " ${idsCL[Yellow]}check${idsCL[Default]} --> Crontab job for maintaining VPN & Media-Services (run */1 min)"
echo -e " ${idsCL[Yellow]}clear${idsCL[Default]} --> Clear all tmp files"
echo -e " ${idsCL[Yellow]}update${idsCL[Default]} --> Check for and install new updates"
echo ""
echo ""
exit 0;;
esac
else
echo -e "${idsCL[Yellow]}Part of the system is currently updating, please try again shortly.\nUpdates usually only take 1-3mins.${idsCL[Default]}"
echo
fi
exit 0