Files
mediamanager/mm-scripts.sh
2023-04-12 19:48:40 -05:00

886 lines
32 KiB
Bash
Executable File

#!/usr/bin/env bash
# MediaManager Scripts
action="$1"
VERBOSE=true
if [ -f /opt/idssys/settings/mediamanager.conf ]; then
mv /opt/idssys/settings/mediamanager.conf ${MMFOLDER}/settings.conf
fi
source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/mediamanager/settings.conf
source /opt/idssys/mediamanager/defaults.inc
[ -f ${MMFOLDER}/vpninfo ] && source ${MMFOLDER}/vpninfo
if [ "${action}" != "check" ] && [ "${action}" != "update" ] && [ "${2}" != "noup" ]; then
if [ -f ${MMFOLDER}/update.available ]; then
dispup="${idsCL[Yellow]} - Update Available! (run '${idsCL[LightCyan]}mm update${idsCL[Yellow]}')${idsCL[Default]}"
else
dispup=""
fi
echo
echo -e "${idsCL[LightGreen]} MediaManager ${idsCL[Default]} ${idsCL[DarkGray]}(ver-${VERS})${dispup}${idsCL[Default]}"
DIVIDER . lightGreen
fi
IFS=,
SERVICES_CHECK=(${SERVICESCHECK})
unset IFS
cw_spc=40
MMSTART(){
[ $VERBOSE = true ] && chktmstart=`date +%s`
if [ "${1}" = "start" ] || [ "${1}" = "restart" ] || [ "${1}" = "" ]; then
GOFORSTART=true;
elif [ "${1}" == "check" ] && [ "$(/sbin/ip link | grep "${VPN_INTERFACE}")" == "" ] && [ ! -f ${MMFOLDER}/mm.vpn* ] && [ ! -f ${MMFOLDER}/vpninfo.* ]; then
GOFORSTART=true;
SENDNOTICE "Starting Media Management Systems" "Fresh Startup"
echo "$(date) - Starting Media Management system - (fresh start)" >> $logfile
echo -e "${idsCL[LightCyan]}Fresh Startup${idsCL[Default]}"
# sleep 2m
elif [ "${1}" = "check" ] && [ "$(/sbin/ip link | grep "${VPN_INTERFACE}")" = "" ] && [ -f ${MMFOLDER}/mm.vpn* ]; then
GOFORSTART=true;
echo "$(date) - Attempting to restart systems after VPN failure..." >> $logfile
echo -e "${idsCL[LightCyan]}Attempting to restart systems after VPN failure...${idsCL[Default]}"
SENDNOTICE "Attempting to start systems after VPN failure..."
else
GOFORSTART=false;
fi
if [ ${GOFORSTART} = true ]; then
mmsv='Starting'
echo
OUT=$(/sbin/ip link | grep "${VPN_INTERFACE}")
if [ ${#OUT} = 0 ]; then
msg="Setting up the VPN Tunnel config"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
privkey=$(/usr/bin/wg genkey)
reqpublickey=$(echo "${privkey}" | /usr/bin/wg pubkey)
reqpublickey="$(php -r 'echo rawurlencode($argv[1]);' -- "$reqpublickey")"
rm -f ${MMFOLDER}/vpninfo
JSON=$(curl https://${VPN_SERVER}:1443/api/v1/setup?public-key=${reqpublickey} --user "${VPN_USER}:${VPN_PASS}" -sk)
echo -e "vpn_publickey=\"$(echo ${JSON} | jq -r '.server_public_key')\"
vpn_server_ipv4=\"$(echo ${JSON} | jq -r '.server_ipv4')\"
vpn_client_ipv4=\"$(echo ${JSON} | jq -r '.client_ipv4')\"
vpn_routes=\"$(echo ${JSON} | jq -r '.routes')\"
vpn_server_address=\"$(echo ${JSON} | jq -r '.vpn_server_address')\"
vpn_server_port=\"$(echo ${JSON} | jq -r '.vpn_server_port')\"
" > ${MMFOLDER}/vpninfo
source ${MMFOLDER}/vpninfo
rm -f /etc/wireguard/${VPN_INTERFACE}.conf
echo -e "[Interface]
PrivateKey = ${privkey}
ListenPort = 55009
MTU = 1390
DNS = 1.1.1.1
Address = ${vpn_client_ipv4}
[Peer]
PublicKey = ${vpn_publickey}
AllowedIPs = 0.0.0.0/0
Endpoint = ${vpn_server_address}:${vpn_server_port}
PersistentKeepalive = 25" >> /etc/wireguard/${VPN_INTERFACE}.conf
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Starting the VPN Tunnel"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
"${VPN_START[@]}" >/dev/null 2>&1
sleep 5s
else
msg="The VPN Tunnel is already running"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
fi
else
msg="Verifying the VPN Tunnel"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
mmsv='Verifying'
fi
[ $VERBOSE = true ] && echo "$(date) - ${mmsv} VPN..." >> $logfile
OUT=$(/sbin/ip link | grep "$VPN_INTERFACE")
if [ ${#OUT} = 0 ]; then
if [ -f ${MMFOLDER}/mm.vpn.fail ]; then
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
echo "$(date) - VPN Failure #2 - restarting system" >> $logfile
SENDNOTICE "VPN Failure" "2nd Failure, will try and restart the VPN" 1
touch ${MMFOLDER}/mm.vpn.fail2
rm -f ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 5m
# "${VPN_START[@]}" >/dev/null 2>&1
# sleep 10s
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
elif [ -f ${MMFOLDER}/mm.vpn.fail2 ]; then
echo "$(date) - VPN Failure #3 - Shutting down Media Management systems" >> $logfile
SENDNOTICE "VPN Failure" "Final Failure, shutting down system" 2
MMSTOP
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
else
echo -e "${idsCL[Red]} failed, restarting system${idsCL[Default]}"
echo "$(date) - VPN Failure #1 - restarting system" >> $logfile
SENDNOTICE "VPN Failure" "1st Failure, will try and restart the VPN" 1
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 10s
# "${VPN_START[@]}" >/dev/null 2>&1
# sleep 10s
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
else
if [ ${GOFORSTART} = true ] || [ "${1}" == "check" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
fi
zerogw=$(/sbin/ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(/sbin/ip route show | grep default | cut -d' ' -f 3)
# if [ "${zerogw}" != "${defaultgw}" ]; then
# if [ "${defaultgw}" != "${VPN_INTERFACE}" ]; then
if [ "${zerogw}" != "${defaultgw}" ] || [ "${defaultgw}" != "${VPN_INTERFACE}" ]; then
echo "$(date) - Adding VPN default routes" >> $logfile
msg="Adding VPN default routes"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
/bin/ip route del default
/bin/ip route add default dev ${VPN_INTERFACE}
/bin/ip route add 0.0.0.0/1 dev ${VPN_INTERFACE}
/bin/ip route add 128.0.0.0/1 dev ${VPN_INTERFACE}
/bin/ip route add ${vpn_server_address}/32 via 10.10.0.1 dev eth0
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
zerogw=$(/sbin/ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(/sbin/ip route show | grep default | cut -d' ' -f 3)
fi
msg="Verifying default network routes"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
checked=false
cc=0
until [ "${checked}" = "" ]; do
if [ "${zerogw}" = "${defaultgw}" ]; then
checked=""
fi
if [ "${checked}" == "false" ] && [ ${cc} -eq 10 ]; then
echo -e "${idsCL[Red]}Could not confirm default routes, restarting system!${idsCL[Default]}"
echo "$(date) - Could not confirm default routes, restarting system" >> $logfile
SENDNOTICE "Network Failure" "Default route failure, will try and restart the VPN" 1
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 10s
# "${VPN_START[@]}" >/dev/null 2>&1
# sleep 10s
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
((cc=${cc}+1))
sleep 1s
zerogw=$(/sbin/ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(/sbin/ip route show | grep default | cut -d' ' -f 3)
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Verifying traffic routing through VPN"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
scip=""
mmip=""
cc=0
until [ "${scip}" != "" ] && [ "${mmip}" != "" ] && [[ "${scip}" != *"connection timed out"* ]] && [[ "${mmip}" != *"connection timed out"* ]]; do
scip=$(/usr/bin/dig +short +time=2 sc1.scity.us @10.10.10.5 -p 5053)
# mmip=$(/usr/bin/dig +short myip.opendns.com @resolver1.opendns.com)
mmip=$(/usr/bin/dig +short +time=2 myip.opendns.com @208.67.222.222)
if [ ${cc} -eq 10 ]; then
echo
echo -e "${idsCL[Red]}Cannot get an WAN address, restarting system!${idsCL[Default]}"
echo "$(date) - Cannot get an WAN address, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
SENDNOTICE "Network Failure" "Cannot get an WAN address, will try and restart the VPN" 1
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP >/dev/null 2>&1
sleep 10s
# "${VPN_START[@]}" >/dev/null 2>&1
# sleep 10s
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
((cc=${cc}+1))
sleep 1s
done
if [ "${scip}" != "${mmip}" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
[ $VERBOSE = true ] && echo "$(date) - WAN Address: ${mmip} - GW Address: ${defaultgw}" >> $logfile
else
echo
echo -e "${idsCL[Red]}Traffic is not routing through VPN, restarting system!${idsCL[Default]}"
echo "$(date) - Traffic is not routing through VPN, restarting system! scip(${scip}) mmip(${mmip})" >> $logfile
[ $VERBOSE = true ] && echo "$(date) - Default Routes: zerogw(${zerogw}) defaultgw(${defaultgw})" >> $logfile
SENDNOTICE "Network Failure" "Traffic route failure, will try and restart the VPN" 1
touch ${MMFOLDER}/mm.vpn.fail
MMSTOP
sleep 10s
# "${VPN_START[@]}" >/dev/null 2>&1
# sleep 10s
rm -f ${MMFOLDER}/mm.stop*
[ $VERBOSE = true ] && DEBUGLOGSTOP
exit 1
fi
msg="Verifying Plex Media is Mounted"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
### CHECK FOR PLEX MOUNT
files=$(shopt -s nullglob dotglob; echo ${PLEX_MOUNT}/*)
if (( ! ${#files} )); then
echo "$(date) - Plex Media Not Mounted, will try and fix" >> $logfile
umount ${PLEX_MOUNT}
sleep 2s
mount -a
sleep 2s
files=$(shopt -s nullglob dotglob; echo ${PLEX_MOUNT}/*)
if (( ! ${#files} )); then
if [ -f ${MMFOLDER}/mm.plexmount.fail ]; then
fd=$(( `date +%s` - `stat -L --format %Y ${MMFOLDER}/mm.plexmount.fail` ))
if [ ${fd} -gt 3600 ]; then
PMFS=true
else
PMFS=false
fi
else
PMFS=true
fi
if [ "${EMAIL_NOTICE}" != "" ] && [ ${PMFS} = true ]; then
SENDNOTICE "Plex Media Not Mounted" "Could not mount the Plex Media Folder" 1
echo "$(date) - Plex Media could not be mounted" >> $logfile
fi
touch ${MMFOLDER}/mm.plexmount.fail
echo -e "${idsCL[Red]}Not Mounted, stopping services${idsCL[Default]}"
for srvc in "${SERVICES_CHECK[@]}"; do
/bin/systemctl stop ${srvc}
done
else
echo -e "${idsCL[Green]}Mounted, fixed${idsCL[Default]}"
rm -f ${MMFOLDER}/mm.plexmount.fail
echo "$(date) - Plex Media now mounted, stopping services to restart them" >> $logfile
for srvc in "${SERVICES_CHECK[@]}"; do
/bin/systemctl stop ${srvc}
done
fi
else
echo -e "${idsCL[Green]}Mounted${idsCL[Default]}"
rm -f ${MMFOLDER}/mm.plexmount.fail
fi
[ "${1}" == "start" ] && [ "${2}" != "vpn" ] && rm -f ${MMFOLDER}/mm.noservices
if [ "${2}" != "vpn" ] && [ ! -f ${MMFOLDER}/mm.noservices ] && [ ! -f ${MMFOLDER}/mm.plexmount.fail ]; then
msg="${mmsv} Services"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
for srvc in "${SERVICES_CHECK[@]}"; do
[ "$(systemctl is-active ${srvc})" != "active" ] && /bin/systemctl restart ${srvc}
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
[ "${1}" == "start" ] || [ "${1}" == "restart" ] && echo
msg="Waiting for Services to Start"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
for srvc in "${SERVICES_CHECK[@]}"; do
if [ "${MM_SERVICE_PORTS[${srvc}]}" != "" ]; then
checkport=$(lsof -Pi :${MM_SERVICE_PORTS[${srvc}]} -sTCP:LISTEN -t)
tryupdate=false
cc=0
until [ "${checkport}" != "" ]; do
if [ ${cc} -eq 10 ]; then
if [ -f ${MMFOLDER}/mm.${srvc}.starting ]; then
msg="${srvc^} Service is not starting properly, will try and update"
echo -e "${idsCL[Red]}${msg}${idsCL[Default]}"
echo "$(date) - ${msg}" >> $logfile
SENDNOTICE "${srvc^} Service Issue" "${msg}" 1
tryupdate="true"
msg="Updating ${srvc^} Service"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
rm -f ${MMFOLDER}/mm.${srvc}.starting
else
[ "${MM_UPDATE_REPO[$srvc]}" != "" ] && touch ${MMFOLDER}/mm.${srvc}.starting
fi
checkport="go"
fi
if [ "${checkport}" != "go" ]; then
checkport=$(lsof -Pi :${MM_SERVICE_PORTS[${srvc}]} -sTCP:LISTEN -t)
((cc=${cc}+1))
sleep 1s
fi
done
if ([ "${checkport}" == "" ] || [ "${tryupdate}" == "true" ]) && [ "${MM_UPDATE_REPO[${srvc}]}" != "" ]; then
UPDATESERVICE ${srvc} >/dev/null 2>&1
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
elif [ "${tryupdate}" == "true" ]; then
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
fi
[ "${checkport}" != "" ] && rm -f ${MMFOLDER}/mm.${srvc}.starting >/dev/null 2>&1
fi
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[LightYellow]}Skipping ${mmsv} Services ${idsCL[Default]}"
fi
echo
if [ "${1}" == "start" ] || [ "${1}" == "restart" ]; then
#echo -e "${idsCL[LightCyan]}It may take a couple of minutes before all services fully start${idsCL[Default]}"
MMSTATUS
fi
if [ -f ${MMFOLDER}/mm.vpn.fail* ]; then
echo "$(date) - VPN issue has been fixed" >> $logfile
SENDNOTICE "Services Restored" "VPN issue has been fixed"
rm -f ${MMFOLDER}/mm.vpn.fail*
fi
rm -f ${MMFOLDER}/mm.stop*
echo -e "${idsCL[Green]}All systems are go!${idsCL[Default]}"
if [ "${1}" = "start" ] || [ "${1}" = "" ]; then
echo "$(date) - Media Management systems started" >> $logfile
fi
fi
echo
[ $VERBOSE = true ] && DEBUGLOGSTOP
#exit 0
}
MMSTOP(){
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${idsCL[Yellow]}The system is already stopped, will verify anyway...${idsCL[Default]}"
else
echo -e "${idsCL[Green]}Stopping System...${idsCL[Default]}"
fi
echo
touch ${MMFOLDER}/mm.stop
# echo -en "${idsCL[LightCyan]}Stopping background MM scripts... ${idsCL[Default]}"
# getMMbash=$(pgrep -f '/mm')
# for i in $getMMbash; do
# /bin/kill $i
# done
# echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Stopping Services"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
for srvc in "${!MM_SERVICES[@]}"; do
if [[ ${srvc} != *"vpn"* ]] && [[ ${srvc} != *"wireguard"* ]]; then
/bin/systemctl stop ${srvc} &
fi
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Waiting for Services to stop"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
for srvc in "${SERVICES_CHECK[@]}"; do
checkport=$(lsof -Pi :${MM_SERVICE_PORTS[${srvc}]} -sTCP:LISTEN -t)
until [ "${checkport}" == "" ]; do
sleep 1s
checkport=$(lsof -Pi :${MM_SERVICE_PORTS[${srvc}]} -sTCP:LISTEN -t)
done
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Shutting Down VPN Tunnel"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
"${VPN_STOP[@]}" >/dev/null 2>&1
checked=false
until [ "${checked}" = "" ]; do
OUT=$(/sbin/ip link | grep "$VPN_INTERFACE")
#OUT=`ip a show $VPN_INTERFACE up` >/dev/null 2>&1
if [ ${#OUT} = 0 ]; then
checked=""
fi
done
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Re-Adding Default Network Routes"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[LightCyan]}${msg}${spc} ${idsCL[White]}: ${idsCL[Default]}"
[ "$(/sbin/ip route show | grep default | cut -d' ' -f 3)" != "10.10.0.1" ] && /bin/ip route add default via 10.10.0.1 && ip route del ${vpn_server_address}
[ $VERBOSE = true ] && echo "$(date) - Added default route back in" >> $logfile
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
rm -f ${MMFOLDER}/vpninfo*
rm -f ${MMFOLDER}/mm.noservices*
rm -f ${MMFOLDER}/mm.stop.checking
echo "$(date) - Media Management systems stopped" >> $logfile
echo
echo -e "${idsCL[Red]}All services have been stopped${idsCL[Default]}"
echo
#exit 0
}
MMSTATUS(){
statusArray=(wireguard)
statusArray+=(${SERVICES_CHECK[@]})
echo
for srvc in "${statusArray[@]}"; do
c=0; spc=''
spc1=$((${cw_spc}-${#MM_SERVICES[${srvc}]}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
dispname="${MM_SERVICES[${srvc}]/(/${idsCL[LightCyan]}(}"
echo -en " ${idsCL[White]}${dispname}${spc}${idsCL[White]}: ${idsCL[Default]}"
if [ "$(systemctl is-active ${srvc})" != "active" ] && [ "${srvc}" != "wireguard" ]; then
if [ -f ${MMFOLDER}/mm.noservices ]; then
echo -e "${idsCL[LightCyan]}Skipping, started wtih VPN only${idsCL[Default]}"
elif [ -f ${MMFOLDER}/mm.plexmount.fail ]; then
echo -e "${idsCL[LightCyan]}Skipping, Plex Media not mounted${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
fi
elif [ "${srvc}" == "wireguard" ] && [ "$(/sbin/ip link | grep $VPN_INTERFACE)" == "" ]; then
echo -e "${idsCL[Red]}Not Running${idsCL[Default]}"
else
[ "${MM_SERVICE_PORTS[${srvc}]}" != "" ] && checkport=$(lsof -Pi :${MM_SERVICE_PORTS[${srvc}]} -sTCP:LISTEN -t)
if ([ "${MM_SERVICE_PORTS[${srvc}]}" != "" ] && [ "${checkport}" != "" ]) || [ "${MM_SERVICE_PORTS[${srvc}]}" == "" ]; then
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
elif [ "${checkport}" == "" ]; then
echo -e "${idsCL[LightMagenta]}Service Running, Web Port Not Accesible${idsCL[Default]}"
fi
if [[ ${srvc} == *"vpn"* ]] || [[ ${srvc} == *"wireguard"* ]]; then
zerogw=$(/sbin/ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(/sbin/ip route show | grep default | cut -d' ' -f 3)
msg=" |- ${idsCL[Default]}0.0.0.0 Gateway"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}+6))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]} : ${idsCL[Default]}"
echo -e "${idsCL[Green]}$zerogw${idsCL[Default]}"
msg=" |- ${idsCL[Default]}DEFAULT Gateway"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}+6))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]} : ${idsCL[Default]}"
if [ "${zerogw}" == "${defaultgw}" ] && [ "${zerogw}" == "${VPN_INTERFACE}" ]; then
echo -e "${idsCL[Green]}$defaultgw${idsCL[Default]}"
else
echo -e "${idsCL[Red]}$defaultgw - Incorrect Gateways${idsCL[Default]}"
if ([ ! -f ${MMFOLDER}/mm.stop* ] && [ "${1}" == "fix" ]) || ([ ! -f ${MMFOLDER}/mm.stop ] && [ -f ${MMFOLDER}/mm.stop.checking ] && [ "${1}" == "fix" ]); then
echo
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 10secs... ${idsCL[Default]}"
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo
echo -e "${idsCL[LightCyan]}Starting system back up...${idsCL[Default]}"
echo
MMSTART
msg=" ${MM_SERVICES[wireguard]}"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]}: ${idsCL[Default]}"
echo -e "${idsCL[Green]}Running${idsCL[Default]}"
zerogw=$(/sbin/ip route show | grep 0.0.0.0 | cut -d' ' -f 3)
defaultgw=$(/sbin/ip route show | grep default | cut -d' ' -f 3)
msg=" |- ${idsCL[Default]}0.0.0.0 Gateway"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}+6))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]} : ${idsCL[Default]}"
echo -e "${idsCL[Green]}$zerogw${idsCL[Default]}"
msg=" |- ${idsCL[Default]}DEFAULT Gateway"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}+6))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]} : ${idsCL[Default]}"
if [ "${zerogw}" == "${defaultgw}" ] && [ "${zerogw}" == "${VPN_INTERFACE}" ]; then
echo -e "${idsCL[Green]}$defaultgw${idsCL[Default]}"
else
echo -e "${idsCL[Red]}$defaultgw - Incorrect Gateways${idsCL[Default]}"
echo
if [ ! -f ${MMFOLDER}/mm.stop.checking ]; then
echo -e "${idsCL[LightCyan]}Shutting down system... fix yo shit${idsCL[Default]}"
MMSTOP
fi
fi
fi
fi
msg=" |- ${idsCL[Default]}Verify routing through VPN"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}+6))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc}${idsCL[White]} : ${idsCL[Default]}"
scip=$(/usr/bin/dig +short sc1.scity.us @10.10.10.5 -p 5053)
# mmip=$(/usr/bin/dig +short myip.opendns.com @resolver1.opendns.com)
mmip=$(/usr/bin/dig +short +time=2 myip.opendns.com @208.67.222.222)
if [ "${scip}" != "${mmip}" ]; then
echo -e "${idsCL[Green]}Good${idsCL[Default]}"
else
echo -e "${idsCL[Red]}Not Routing${idsCL[Default]}"
if ([ ! -f ${MMFOLDER}/mm.stop* ] && [ "${1}" == "fix" ]) || ([ ! -f ${MMFOLDER}/mm.stop ] && [ -f ${MMFOLDER}/mm.stop.checking ] && [ "${1}" == "fix" ]); then
echo
echo -e "${idsCL[LightCyan]}Restarting system...${idsCL[Default]}"
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 10secs... ${idsCL[Default]}"
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo
echo -e "${idsCL[LightCyan]}Starting system back up...${idsCL[Default]}"
echo
MMSTART
fi
fi
echo
fi
fi
done
echo
echo
}
GETVPNINFO(){
echo -e "${idsCL[Green]}VPN Connection Info${idsCL[Default]}"
echo
msg="Server Network"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -n "${idsCL[LightCyan]}${msg}${spc}${idsCL[White]}: ${idsCL[Green]}${VPN_SERVER}${idsCL[Default]}"
}
UPDATESERVICE(){
if [ "${1}" != "" ]; then
usrvc=${1}
echo "${usrvc} Update" > ${MMFOLDER}/mm.update.stop
echo "$(date) - Updating Service (${usrvc}) : Starting" >> $logfile
echo
echo -e "${idsCL[Green]}Updating ${usrvc^} Service${idsCL[Default]}"
DIVIDER . lightGreen
msg="Stopping service and preparing"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc} : ${idsCL[Default]}"
rm -f ${MMFOLDER}/logs/${usrvc}.update
touch ${MMFOLDER}/logs/${usrvc}.update
/bin/systemctl stop ${usrvc} >> ${MMFOLDER}/logs/${usrvc}.update
[ "${usrvc}" == "radarr" ] && /bin/systemctl stop radarr3d >> ${MMFOLDER}/logs/${usrvc}.update
tmpfile=/tmp/${usrvc}-update.tar.gz
bak=/opt/${usrvc}.bak.tgz
[[ -f ${bak} ]] && rm -f "${bak}" &>/dev/null
[[ -f ${tmpfile} ]] && rm -f "${tmpfile}" &>/dev/null
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
msg="Finding latest update url"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc} : ${idsCL[Default]}"
if [[ $BETA_SERVICES == *${usrvc}* ]]; then
latest_release_url=`curl -s https://api.github.com/repos/${MM_UPDATE_REPO[${usrvc}]}/${usrvc}/releases | jq -r '.[0].assets[].browser_download_url' | grep "${MM_UPDATE_REPO_GREP[${usrvc}]}" | cut -d '"' -f 4 | grep -i 'https://'`
echo "$(date) - Updating Service (${usrvc}) : BETA UPDATEURL = ${latest_release_url}" >> $logfile
beta="BETA "
else
latest_release_url=`curl -s https://api.github.com/repos/${MM_UPDATE_REPO[${usrvc}]}/${usrvc}/releases/latest | jq -r '.assets[].browser_download_url' | grep "${MM_UPDATE_REPO_GREP[${usrvc}]}" | cut -d '"' -f 4 | grep -i 'https://'`
echo "$(date) - Updating Service (${usrvc}) : UPDATEURL = ${latest_release_url}" >> $logfile
beta=""
fi
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
echo -e "${idsCL[LightCyan]}${beta}Download URL: ${idsCL[LightGreen]}${latest_release_url}${idsCL[Default]}"
echo
msg="Downloading and updating"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc} : ${idsCL[Default]}"
if curl --output /dev/null --silent --head --fail $latest_release_url; then
wget -qO ${tmpfile} $latest_release_url >> ${MMFOLDER}/logs/${usrvc}.update
if [[ ( $? -eq 0 ) && ( -f ${tmpfile} ) ]]; then
if [ "${usrvc}" != "bazarr" ]; then
rm -Rf /opt/${usrvc^} >> ${MMFOLDER}/logs/${usrvc}.update
mkdir /opt/${usrvc^} >> ${MMFOLDER}/logs/${usrvc}.update
fi
/bin/tar -cpPf $bak /opt/${usrvc^} >> ${MMFOLDER}/logs/${usrvc}.update
if [ "${usrvc}" == "ombi" ]; then
/bin/tar xfP ${tmpfile} -C /opt/${usrvc^} >> ${MMFOLDER}/logs/${usrvc}.update
elif [ "${usrvc}" == "bazarr" ]; then
/usr/bin/unzip -oqq ${tmpfile} -d /opt/${usrvc^}
else
/bin/tar xfP ${tmpfile} -C /opt >> ${MMFOLDER}/logs/${usrvc}.update
fi
chown -R user:users /opt/${usrvc^} >> ${MMFOLDER}/logs/${usrvc}.update
fi
fi
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
rm -f ${tmpfile} &>/dev/null
msg="Starting service back up"
c=0; spc=''; spc1=$((${cw_spc}-${#msg}))
until [ $c = ${spc1} ]; do spc="${spc} "; c=`expr $c + 1`; done
echo -en "${idsCL[White]}${msg}${spc} : ${idsCL[Default]}"
OUT=$(ip link | grep "$VPN_INTERFACE") >/dev/null 2>&1
if [ ${#OUT} -ne 0 ] && [ ! -f ${MMFOLDER}/mm.stop ] && [ ! -f ${MMFOLDER}/mm.noservices ] && [[ " ${SERVICES_CHECK[*]} " =~ " ${usrvc} " ]]; then
/bin/systemctl start ${usrvc} >> ${MMFOLDER}/logs/${usrvc}.update
[ "${usrvc}" == "radarr" ] && /bin/systemctl start radarr3d >> ${MMFOLDER}/logs/${usrvc}.update
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
else
echo -e "${idsCL[LightCyan]}Skipping${idsCL[Default]}"
fi
rm -f ${MMFOLDER}/mm.update.stop
echo "$(date) - Updating Service (${usrvc}) : Completed" >> $logfile
echo
echo -e "${idsCL[Green]}${usrvc} Service has been updated${idsCL[Default]}"
else
echo -e "${idsCL[Red]}No service selected for update${idsCL[Default]}"
fi
}
SHOWSRVCUPDATES(){
echo -e "${idsCL[LightCyan]}Latest Update URLs Available${idsCL[Default]}"
DIVIDER . lightGreen
for usrvc in "${UPDATE_SERVICES[@]}"; do
beta_url=`curl -s https://api.github.com/repos/${MM_UPDATE_REPO[${usrvc}]}/${usrvc}/releases | jq -r '.[0].assets[].browser_download_url' | grep "${MM_UPDATE_REPO_GREP[${usrvc}]}" | cut -d '"' -f 4 | grep -i 'https://' | cut -d/ -f8`
#beta_url=${beta_url} | cut -d/ -f8
latest_url=`curl -s https://api.github.com/repos/${MM_UPDATE_REPO[${usrvc}]}/${usrvc}/releases/latest | jq -r '.assets[].browser_download_url' | grep "${MM_UPDATE_REPO_GREP[${usrvc}]}" | cut -d '"' -f 4 | grep -i 'https://' | cut -d/ -f8`
#latest_url=${latest_url} | cut -d/ -f8
[[ $BETA_SERVICES == *${usrvc}* ]] && cur='Beta' || cur='Stable'
echo -e "${idsCL[LightCyan]}${MM_SERVICES[${usrvc}]}:${idsCL[Default]} ${cur} Selected"
echo -e "${idsCL[LightGreen]}Stable Release : ${latest_url}${idsCL[Default]}"
echo -e "${idsCL[LightGreen]}Beta Release : ${beta_url}${idsCL[Default]}"
echo
done
}
if [ ! -f ${MMFOLDER}/mm.update.stop ]; then
if [ -f ${MMFOLDER}/mm.stop ]; then
stoppedsince=`date +%s`-$(stat -c %Y ${MMFOLDER}/mm.stop)
stopmsg="${idsCL[Yellow]}The system has currently been stopped for '$(SHOW_TIME ${stoppedsince})'.\n${idsCL[LightCyan]}(You'll need to run \"mm start\" to start services again)${idsCL[Default]}"
fi
case $action in
start | check)
if [ -f ${MMFOLDER}/mm.stop ] && [ "${action}" == "check" ]; then
echo -e "${stopmsg}"
# [ $VERBOSE = true ] && echo "$(date) - Not checking, system has been stopped" >> $logfile
echo
exit 1
elif [ "${VPN_USER}" == "" ] || [ "${VPN_PASS}" == "" ] || [ "${VPN_SERVER}" == "" ]; then
echo "$(date) - VPN User Info Missing" >> $logfile
echo -e "${idsCL[Red]}VPN User Info Missing${idsCL[Default]}"
echo
exit 1
elif [ ! -f ${MMFOLDER}/mm.stop ] && [ -f ${MMFOLDER}/mm.stop.checking ]; then
checkstopped=`expr $(date +%s) - $(stat -c %Y ${MMFOLDER}/mm.stop.checking)`
if [ $checkstopped -gt 900 ]; then
rm -f ${MMFOLDER}/mm.stop*
echo "$(date) - Bash script stuck in stop mode after doing a check, will restart" >> $logfile
SENDNOTICE "MM Script Error" "Bash script stuck in check mode, restarting system"
MMSTOP
rm -f ${MMFOLDER}/mm.stop*
getStuckMM=$(pgrep -f '/mm')
for i in $getStuckMM; do
/bin/kill $i
done
else
exit 1
fi
fi
[ "${action}" != "status" ] && touch ${MMFOLDER}/mm.stop.checking
[ "${2}" == "vpn" ] && touch ${MMFOLDER}/mm.noservices
[ ! -f ${MMFOLDER}/mm.stop ] && MMSTART ${action} ${2}
[ "${action}" != "status" ] && rm -f ${MMFOLDER}/mm.stop.checking
;;
stop)
MMSTOP
;;
restart)
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${stopmsg}"
echo
exit 1
fi
MMSTOP
echo -en "${idsCL[LightCyan]}Pausing for 10secs... ${idsCL[Default]}"
sleep 10s
echo -e "${idsCL[Green]}Done${idsCL[Default]}"
MMSTART ${action}
;;
status)
if [ -f ${MMFOLDER}/mm.stop ]; then
echo -e "${stopmsg}"
echo
fi
MMSTATUS ${2}
;;
vpninfo)
GETVPNINFO
;;
update);;
update-service)
UPDATESERVICE ${2}
;;
update-allservices)
for usrvc in "${SERVICES_CHECK[@]}"; do
[ "${MM_UPDATE_REPO[${usrvc}]}" != "" ] && UPDATESERVICE ${usrvc}
done
;;
update-list)
SHOWSRVCUPDATES
;;
clear) echo -e "${idsCL[LightCyan]}Clearing all mm.* files ${idsCL[Default]}"
echo
;;
*)
echo -e " Usage: ${idsCL[LightCyan]}mm {option}${idsCL[Default]}"
echo
echo -e " ${idsCL[Yellow]}start${idsCL[Default]} --> Start services"
echo -e " ${idsCL[Yellow]}stop${idsCL[Default]} --> Stop services"
echo -e " ${idsCL[Yellow]}restart${idsCL[Default]} --> Restart services"
echo -e " ${idsCL[Yellow]}update-allservices ${idsCL[Default]} --> Update all services"
echo -e " ${idsCL[Yellow]}update-service <service>${idsCL[Default]} --> Update specified service"
echo -e " ${idsCL[Yellow]}update-list${idsCL[Default]} --> List available releases"
echo
echo -e " ${idsCL[Yellow]}status${idsCL[Default]} --> Check Status on VPN and Media Services"
echo -e " ${idsCL[Yellow]}vpninfo${idsCL[Default]} --> Show VPN Info"
echo
echo -e " ${idsCL[Yellow]}check${idsCL[Default]} --> Crontab job for maintaining VPN & Media Services (run */1 min)"
echo -e " ${idsCL[Yellow]}clear${idsCL[Default]} --> Clear all tmp files"
echo -e " ${idsCL[Yellow]}update${idsCL[Default]} --> Check for and install new updates"
echo ""
echo ""
exit 0;;
esac
else
echo -e "${idsCL[Yellow]}Part of the system is currently updating, please try again shortly.\nUpdates usually only take 1-3mins.${idsCL[Default]}"
echo
fi
exit 0