You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
363 lines
13 KiB
363 lines
13 KiB
#!/bin/bash
|
|
|
|
# This script runs on after start in background
|
|
# as a service and gets restarted on failure
|
|
# it runs ALMOST every seconds
|
|
|
|
# INFOFILE - state data from bootstrap
|
|
infoFile="/home/admin/raspiblitz.info"
|
|
|
|
# CONFIGFILE - configuration of RaspiBlitz
|
|
configFile="/mnt/hdd/raspiblitz.conf"
|
|
|
|
# LOGS see: sudo journalctl -f -u background
|
|
|
|
# Check if HDD contains configuration
|
|
configExists=$(ls ${configFile} | grep -c '.conf')
|
|
if [ ${configExists} -eq 1 ]; then
|
|
source ${configFile}
|
|
else
|
|
source ${infoFile}
|
|
fi
|
|
|
|
echo "_background.sh STARTED"
|
|
|
|
# monitor lost LND sync
|
|
syncedSince=0
|
|
lastSyncState=0
|
|
|
|
counter=0
|
|
while [ 1 ]
|
|
do
|
|
|
|
###############################
|
|
# Prepare this loop
|
|
###############################
|
|
|
|
# count up
|
|
counter=$(($counter+1))
|
|
|
|
# gather the uptime seconds
|
|
upSeconds=$(cat /proc/uptime | grep -o '^[0-9]\+')
|
|
|
|
####################################################
|
|
# RECHECK DHCP-SERVER
|
|
# https://github.com/rootzoll/raspiblitz/issues/160
|
|
####################################################
|
|
|
|
# every 5 minutes
|
|
recheckDHCP=$((($counter % 300)+1))
|
|
if [ ${recheckDHCP} -eq 1 ]; then
|
|
echo "*** RECHECK DHCP-SERVER ***"
|
|
|
|
# get the local network IP
|
|
localip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/')
|
|
echo "localip(${localip})"
|
|
|
|
# detect a missing DHCP config
|
|
if [ "${localip:0:4}" = "169." ]; then
|
|
echo "Missing DHCP detected ... trying emergency reboot"
|
|
sudo shutdown -r now
|
|
else
|
|
echo "DHCP OK"
|
|
fi
|
|
|
|
fi
|
|
|
|
####################################################
|
|
# CHECK FOR UNDERVOLTAGE REPORTS
|
|
# every 1 hour scan for undervoltage reports
|
|
####################################################
|
|
recheckUndervoltage=$(($counter % 3600))
|
|
if [ ${recheckUndervoltage} -eq 1 ]; then
|
|
echo "*** RECHECK UNDERVOLTAGE ***"
|
|
countReports=$(sudo cat /var/log/syslog | grep -c "Under-voltage detected!")
|
|
echo "${countReports} undervoltage reports found in syslog"
|
|
if [ ${#undervoltageReports} -eq 0 ]; then
|
|
# write new value to info file
|
|
undervoltageReports="${countReports}"
|
|
echo "undervoltageReports=${undervoltageReports}" >> ${infoFile}
|
|
else
|
|
# update value in info file
|
|
sed -i "s/^undervoltageReports=.*/undervoltageReports=${countReports}/g" ${infoFile}
|
|
fi
|
|
fi
|
|
|
|
####################################################
|
|
# RECHECK PUBLIC IP
|
|
# when public IP changes, restart LND with new IP
|
|
####################################################
|
|
|
|
# every 15min - not too often
|
|
# because its a ping to external service
|
|
recheckPublicIP=$((($counter % 900)+1))
|
|
# prevent when lndAddress is set
|
|
if [ ${#lndAddress} -gt 3 ]; then
|
|
recheckPublicIP=0
|
|
fi
|
|
updateDynDomain=0
|
|
if [ ${recheckPublicIP} -eq 1 ]; then
|
|
echo "*** RECHECK PUBLIC IP ***"
|
|
|
|
# execute only after setup when config exists
|
|
if [ ${configExists} -eq 1 ]; then
|
|
|
|
# get actual public IP
|
|
freshPublicIP=$(curl -s http://v4.ipv6-test.com/api/myip.php 2>/dev/null)
|
|
|
|
# sanity check on IP data
|
|
# see https://github.com/rootzoll/raspiblitz/issues/371#issuecomment-472416349
|
|
echo "-> sanity check of new IP data"
|
|
if [[ $freshPublicIP =~ ^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$ ]]; then
|
|
echo "OK IPv6"
|
|
elif [[ $freshPublicIP =~ ^([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])\.([0-9]{1,2}|1[0-9][0-9]|2[0-4][0-9]|25[0-5])$ ]]; then
|
|
echo "OK IPv4"
|
|
else
|
|
echo "FAIL - not an IPv4 or IPv6 address"
|
|
freshPublicIP=""
|
|
fi
|
|
|
|
if [ ${#freshPublicIP} -eq 0 ]; then
|
|
|
|
echo "freshPublicIP is ZERO - ignoring"
|
|
|
|
# check if changed
|
|
elif [ "${freshPublicIP}" != "${publicIP}" ]; then
|
|
|
|
# 1) update config file
|
|
echo "update config value"
|
|
sed -i "s/^publicIP=.*/publicIP='${freshPublicIP}'/g" ${configFile}
|
|
publicIP='${freshPublicIP}'
|
|
|
|
# 2) only restart LND if dynDNS is activated
|
|
# because this signals that user wants "public node"
|
|
if [ ${#dynDomain} -gt 0 ]; then
|
|
echo "restart LND with new environment config"
|
|
# restart and let to auto-unlock (if activated) do the rest
|
|
sudo systemctl restart lnd.service
|
|
fi
|
|
|
|
# 2) trigger update if dnyamic domain (if set)
|
|
updateDynDomain=1
|
|
|
|
else
|
|
echo "public IP has not changed"
|
|
fi
|
|
|
|
else
|
|
echo "skip - because setup is still running"
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
###############################
|
|
# SCB Monitoring
|
|
###############################
|
|
|
|
# check every 1min
|
|
recheckSCB=$(($counter % 60))
|
|
if [ ${recheckSCB} -eq 1 ]; then
|
|
#echo "SCB Monitoring ..."
|
|
source ${configFile}
|
|
# check if channel.backup exists
|
|
scbExists=$(sudo ls /mnt/hdd/lnd/data/chain/${network}/${chain}net/channel.backup 2>/dev/null | grep -c 'channel.backup')
|
|
if [ ${scbExists} -eq 1 ]; then
|
|
#echo "Found Channel Backup File .. check if changed .."
|
|
md5checksumORG=$(sudo md5sum /mnt/hdd/lnd/data/chain/${network}/${chain}net/channel.backup 2>/dev/null | head -n1 | cut -d " " -f1)
|
|
md5checksumCPY=$(sudo md5sum /home/admin/.lnd/data/chain/${network}/${chain}net/channel.backup 2>/dev/null | head -n1 | cut -d " " -f1)
|
|
if [ "${md5checksumORG}" != "${md5checksumCPY}" ]; then
|
|
echo "--> Channel Backup File changed"
|
|
|
|
# make copy to sd card (as local basic backup)
|
|
sudo mkdir -p /home/admin/.lnd/data/chain/${network}/${chain}net/ 2>/dev/null
|
|
sudo cp /mnt/hdd/lnd/data/chain/${network}/${chain}net/channel.backup /home/admin/.lnd/data/chain/${network}/${chain}net/channel.backup
|
|
echo "OK channel.backup copied to '/home/admin/.lnd/data/chain/${network}/${chain}net/channel.backup'"
|
|
|
|
# check if a SCP backup target is set
|
|
# paramter in raspiblitz.conf:
|
|
# scpBackupTarget='[USER]@[SERVER]:[DIRPATH-WITHOUT-ENDING-/]'
|
|
# On target server add the public key of your RaspiBlitz to the authorized_keys for the user
|
|
# https://www.linode.com/docs/security/authentication/use-public-key-authentication-with-ssh/
|
|
if [ ${#scpBackupTarget} -gt 0 ]; then
|
|
echo "--> Offsite-Backup SCP Server"
|
|
# its ok to ignore known host, because data is encrypted (worst case of MiM would be: no offsite channel backup)
|
|
# but its more likely that whithout ignoriing known host, script might not run thru and that way: no offsite channel backup
|
|
sudo scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null /home/admin/.lnd/data/chain/${network}/${chain}net/channel.backup ${scpBackupTarget}/channel.backup
|
|
result=$?
|
|
if [ ${result} -eq 0 ]; then
|
|
echo "OK - SCP Backup exited with 0"
|
|
else
|
|
echo "FAIL - SCP Backup exited with ${result}"
|
|
fi
|
|
fi
|
|
|
|
# check if a DropBox backup target is set
|
|
# paramter in raspiblitz.conf:
|
|
# dropboxBackupTarget='[DROPBOX-APP-OAUTH2-TOKEN]'
|
|
# see dropbox setup: https://gist.github.com/vindard/e0cd3d41bb403a823f3b5002488e3f90
|
|
if [ ${#dropboxBackupTarget} -gt 0 ]; then
|
|
echo "--> Offsite-Backup Dropbox"
|
|
source <(sudo /home/admin/config.scripts/dropbox.upload.sh upload ${dropboxBackupTarget} /home/admin/.lnd/data/chain/${network}/${chain}net/channel.backup)
|
|
if [ ${#err} -gt 0 ]; then
|
|
echo "FAIL - ${err}"
|
|
echo "${errMore}"
|
|
else
|
|
echo "OK - ${upload}"
|
|
fi
|
|
fi
|
|
|
|
#else
|
|
# echo "Channel Backup File not changed."
|
|
fi
|
|
#else
|
|
# echo "No Channel Backup File .."
|
|
fi
|
|
fi
|
|
|
|
###############################
|
|
# LND MONITOR LOST SYNC
|
|
###############################
|
|
|
|
# check every 5min
|
|
recheckSync=$(($counter % 300))
|
|
if [ ${recheckSync} -eq 1 ]; then
|
|
source ${configFile}
|
|
#echo "LND MONITOR LOST SYNC ..."
|
|
lndSynced=$(sudo -u bitcoin /usr/local/bin/lncli --chain=${network} --network=${chain}net getinfo 2>/dev/null | jq -r '.synced_to_chain' | grep -c true)
|
|
#echo "lndSynced(${lndSynced})"
|
|
#echo "syncedSince(${syncedSince})"
|
|
#echo "lastSyncState(${lastSyncState})"
|
|
if [ ${lndSynced} -eq ${lastSyncState} ]; then
|
|
|
|
echo "no sync change"
|
|
if [ ${lndSynced} -eq 1 ]; then
|
|
#echo "all is good - LND still in sync now for:"
|
|
actualSecondsTimestamp=$(date +%s)
|
|
secondsInSync=$(echo "${actualSecondsTimestamp}-"${syncedSince} | bc)
|
|
#echo "${secondsInSync} seconds"
|
|
|
|
# when >10min in sync
|
|
if [ ${secondsInSync} -gt 3600 ]; then
|
|
#echo "LND in sync for longer then 1 hour"
|
|
if [ "${backupTorrentSeeding}" == "on" ]; then
|
|
#echo "Backup Torrent Seeding is ON - check if already running"
|
|
source <(sudo -u admin /home/admin/50torrentHDD.sh status)
|
|
if [ "${baseSeeding}" == "0" ] || [ "${updateSeeding}" == "0" ]; then
|
|
echo "---> STARTING Backup Torrent Seeding"
|
|
sudo -u admin /home/admin/50torrentHDD.sh backup-torrent-hosting
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
else
|
|
#echo "still not in sync"
|
|
if [ ${syncedSince} -gt 0 ]; then
|
|
|
|
echo "was in sync at least once since rinning but lost now for:"
|
|
actualSecondsTimestamp=$(date +%s)
|
|
secondsOutOfSync=$(echo "${actualSecondsTimestamp}-"${syncedSince} | bc)
|
|
echo "${secondsOutOfSync} seconds"
|
|
|
|
# when >1h out of sync
|
|
#if [ ${secondsOutOfSync} -gt 3600 ]; then
|
|
# echo "!!!! LND fell out of sync for longer then 1 hour !!!"
|
|
# TODO: When auto-unlock is ON --> consider implementing restart (this sometimes help)
|
|
#fi
|
|
|
|
fi
|
|
fi
|
|
|
|
else
|
|
echo "sync change detected"
|
|
if [ ${lastSyncState} -eq 1 ] && [ ${lndSynced} -eq 0 ]; then
|
|
echo "--> LND SYNC LOST"
|
|
|
|
if [ "${backupTorrentSeeding}" == "on" ]; then
|
|
#echo "Backup Torrent Seeding is ON - check if still running"
|
|
source <(sudo -u admin /home/admin/50torrentHDD.sh status)
|
|
if [ "${baseSeeding}" == "1" ] || [ "${updateSeeding}" == "1" ]; then
|
|
echo "---> STOPPING Backup Torrent Seeding"
|
|
sudo -u admin /home/admin/50torrentHDD.sh stop
|
|
fi
|
|
fi
|
|
|
|
else
|
|
if [ ${syncedSince} -eq 0 ]; then
|
|
echo "--> LND SYNC GAINED"
|
|
else
|
|
echo "--> LND SYNC RECOVERED"
|
|
fi
|
|
syncedSince=$(date +%s)
|
|
fi
|
|
fi
|
|
|
|
lastSyncState=${lndSynced}
|
|
fi
|
|
|
|
###############################
|
|
# LND AUTO-UNLOCK
|
|
###############################
|
|
|
|
# check every 10secs
|
|
recheckAutoUnlock=$((($counter % 10)+1))
|
|
if [ ${recheckAutoUnlock} -eq 1 ]; then
|
|
|
|
# check if auto-unlock feature if activated
|
|
if [ "${autoUnlock}" = "on" ]; then
|
|
|
|
# check if lnd is locked
|
|
locked=$(sudo -u bitcoin /usr/local/bin/lncli --chain=${network} --network=${chain}net getinfo 2>&1 | grep -c unlock)
|
|
if [ ${locked} -gt 0 ]; then
|
|
|
|
echo "STARTING AUTO-UNLOCK ..."
|
|
|
|
# building REST command
|
|
passwordC=$(sudo cat /root/lnd.autounlock.pwd)
|
|
command="sudo python /home/admin/config.scripts/lnd.unlock.py '${passwordC}'"
|
|
bash -c "${command}"
|
|
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
###############################
|
|
# UPDATE DYNAMIC DOMAIN
|
|
# like afraid.org
|
|
# ! experimental
|
|
###############################
|
|
|
|
# if not activated above, update every 6 hours
|
|
if [ ${updateDynDomain} -eq 0 ]; then
|
|
# dont +1 so that it gets executed on first loop
|
|
updateDynDomain=$(($counter % 21600))
|
|
fi
|
|
if [ ${updateDynDomain} -eq 1 ]; then
|
|
echo "*** UPDATE DYNAMIC DOMAIN ***"
|
|
# check if update URL for dyn Domain is set
|
|
if [ ${#dynUpdateUrl} -gt 6 ]; then
|
|
# calling the update url
|
|
echo "calling: ${dynUpdateUrl}"
|
|
echo "to update domain: ${dynDomain}"
|
|
curl --connect-timeout 6 ${dynUpdateUrl}
|
|
else
|
|
echo "'dynUpdateUrl' not set in ${configFile}"
|
|
fi
|
|
fi
|
|
|
|
###############################
|
|
# Prepare next loop
|
|
###############################
|
|
|
|
# sleep 1 sec
|
|
sleep 1
|
|
|
|
# limit counter to max seconds per week:
|
|
# 604800 = 60sec * 60min * 24hours * 7days
|
|
if [ ${counter} -gt 604800 ]; then
|
|
counter=0
|
|
echo "counter zero reset"
|
|
fi
|
|
|
|
done
|
|
|
|
|