#!/bin/bash source /home/admin/raspiblitz.info source /mnt/hdd/raspiblitz.conf # command info if [ "$1" = "-h" ] || [ "$1" = "-help" ]; then echo "# script to scan the state of the system after setup" exit 1 fi # measure time of scan startTime=$(date +%s) # macke sure temp folder on HDD is available and fro all usable sudo mkdir /mnt/hdd/temp 2>/dev/null sudo chmod 777 -R /mnt/hdd/temp 2>/dev/null # localIP localip=$(ip addr | grep 'state UP' -A2 | tail -n1 | awk '{print $2}' | cut -f1 -d'/') echo "localIP='${localip}'" # temp - no measurement in a VM tempC=0 if [ -d "/sys/class/thermal/thermal_zone0/" ]; then tempC=$(echo "scale=1; $(cat /sys/class/thermal/thermal_zone0/temp)/1000" | bc) echo "tempCelsius='${tempC}'" fi # uptime in seconds uptime=$(awk '{printf("%d\n",$1 + 0.5)}' /proc/uptime) echo "uptime=${uptime}" # count restarts of bitcoind/litecoind startcountBlockchain=$(cat /home/admin/systemd.blockchain.log 2>/dev/null | grep -c "STARTED") echo "startcountBlockchain=${startcountBlockchain}" # is bitcoind running bitcoinRunning=$(systemctl status ${network}d.service 2>/dev/null | grep -c running) echo "bitcoinActive=${bitcoinRunning}" if [ ${bitcoinRunning} -eq 1 ]; then # get blockchain info sudo -u bitcoin ${network}-cli -datadir=/home/bitcoin/.${network} getblockchaininfo 1>/mnt/hdd/temp/.bitcoind.out 2>/mnt/hdd/temp/.bitcoind.error # check if error on request blockchaininfo=$(cat /mnt/hdd/temp/.bitcoind.out 2>/dev/null) bitcoinError=$(cat /mnt/hdd/temp/.bitcoind.error 2>/dev/null) #rm /mnt/hdd/temp/.bitcoind.error 2>/dev/null if [ ${#bitcoinError} -gt 0 ]; then bitcoinErrorShort=$(echo ${bitcoinError/error*:/} | sed 's/[^a-zA-Z0-9 ]//g') echo "bitcoinErrorShort='${bitcoinErrorShort}'" bitcoinErrorFull=$(echo ${bitcoinError} | tr -d "'") echo "bitcoinErrorFull='${bitcoinErrorFull}'" else ############################## # Get data from blockchaininfo ############################## # get total number of blocks total=$(echo ${blockchaininfo} | jq -r '.blocks') echo "blockchainHeight=${total}" # is initial sync of blockchain initialSync=$(echo ${blockchaininfo} | jq -r '.initialblockdownload' | grep -c 'true') echo "initialSync=${initialSync}" # get blockchain sync progress syncProgress="$(echo ${blockchaininfo} | jq -r '.verificationprogress')" syncProgress=$(echo $syncProgress | awk '{printf( "%.2f%%", 100 * $1)}' | tr '%' ' ' | tr -s " ") echo "syncProgress=${syncProgress}" fi else # find out why Bitcoin not running pathAdd="" if [ "${chain}" = "test" ]; then pathAdd="/testnet3" fi #### POSSIBLE/SOFT PROBLEMS # place here in future analysis #### HARD PROBLEMS # LOW DISK SPACE lowDiskSpace=$(sudo tail -n 100 /mnt/hdd/${network}${pathAdd}/debug.log 2>/dev/null | grep -c "Error: Disk space is low!") if [ ${lowDiskSpace} -gt 0 ]; then bitcoinErrorShort="HDD DISK SPACE LOW" bitcoinErrorFull="HDD DISK SPACE LOW - check what data you can delete on HDD and restart" fi #### GENERIC ERROR FIND # if still no error identified - search logs for generic error (after 4min uptime) if [ ${#bitcoinErrorShort} -eq 0 ] && [ ${uptime} -gt 240 ]; then bitcoinErrorFull=$(sudo tail -n 100 /mnt/hdd/${network}${pathAdd}/debug.log 2>/dev/null | grep -c "Error:" | tail -1 | tr -d "'") if [ ${#bitcoinErrorFull} -gt 0 ]; then bitcoinErrorShort="Error found in Logs" fi fi # output error if found if [ ${#bitcoinErrorShort} -gt 0 ]; then echo "bitcoinErrorShort='${bitcoinErrorShort}'" echo "bitcoinErrorFull='${bitcoinErrorFull}'" /home/admin/config.scripts/blitz.systemd.sh log blockchain "ERROR: ${bitcoinErrorShort}" fi fi # count restarts of bitcoind/litecoind startcountLightning=$(cat /home/admin/systemd.lightning.log 2>/dev/null | grep -c "STARTED") echo "startcountLightning=${startcountLightning}" # is LND running lndRunning=$(systemctl status lnd.service 2>/dev/null | grep -c running) echo "lndActive=${lndRunning}" if [ ${lndRunning} -eq 1 ]; then # get LND info lndRPCReady=1 lndinfo=$(sudo -u bitcoin lncli getinfo 2>/mnt/hdd/temp/.lnd.error) # check if error on request lndErrorFull=$(cat /mnt/hdd/temp/.lnd.error 2>/dev/null) lndErrorShort='' #rm /mnt/hdd/temp/.lnd.error 2>/dev/null if [ ${#lndErrorFull} -gt 0 ]; then # flag if error could be resoled by analysis errorResolved=0 ### analyse LND logs since start # find a the line number in logs of start of LND # just do this on error case to save on processing memory lndStartLineNumber=$(sudo cat /mnt/hdd/lnd/logs/${network}/${chain}net/lnd.log 2>/dev/null | grep -in "LTND: Active chain:" | tail -1 | cut -d ":" -f1) # get logs of last LND start lndLogsAfterStart=$(sudo tail --lines=+${lndStartLineNumber} /mnt/hdd/lnd/logs/${network}/${chain}net/lnd.log 2>/dev/null) # check RPC server ready (can take some time after wallet was unlocked) lndRPCReady=$(echo "${lndLogsAfterStart}" | grep -c "RPCS: RPC server listening on") echo "lndRPCReady=${lndRPCReady}" # check wallet if wallet was opened (after correct password) lndWalletOpened=$(echo "${lndLogsAfterStart}" | grep -c "LNWL: Opened wallet") echo "walletOpened=${lndWalletOpened}" # check wallet if wallet is ready (can take some time after wallet was opened) lndWalletReady=$(echo "${lndLogsAfterStart}" | grep -c "LTND: LightningWallet opened") echo "walletReady=${lndWalletReady}" ### check errors # scan error for walletLocked as common error locked=$(echo ${lndErrorFull} | grep -c 'Wallet is encrypted') if [ ${locked} -gt 0 ]; then echo "walletLocked=1" else echo "walletLocked=0" rpcNotWorking=$(echo ${lndErrorFull} | grep -c 'connection refused') if [ ${rpcNotWorking} -gt 0 ]; then # this can happen for a long time when LND is starting fresh sync # on first startup - check if logs since start signaled RPC ready before if [ ${lndRPCReady} -eq 0 ]; then # nullify error - this is normal lndErrorFull="" errorResolved=1 # oputput basic data because no error echo "# LND RPC is still warming up - no scan progress: prepare scan" echo "scanTimestamp=-2" echo "syncedToChain=0" else echo "# LND RPC was started - some other problem going on" lndErrorShort='LND RPC not responding' lndErrorFull=$(echo "LND RPC is not responding. LND may have problems starting up. Check logs, config files and systemd service. Org-Error: ${lndErrorFull}" | tr -d "'") fi fi # if not known error and not resolved before - keep generic if [ ${#lndErrorShort} -eq 0 ] && [ ${errorResolved} -eq 0 ]; then lndErrorShort='Unkown Error - see logs' lndErrorFull=$(echo ${lndErrorFull} | tr -d "'") fi # write to results if [ ${#lndErrorFull} -gt 0 ]; then echo "lndErrorShort='${lndErrorShort}'" echo "lndErrorFull='${lndErrorFull}'" /home/admin/config.scripts/blitz.systemd.sh log lightning "ERROR: ${lndErrorFull}" fi fi else # check if wallet is locked locked=$(echo ${lndinfo} | grep -c unlock) if [ ${locked} -gt 0 ]; then echo "walletLocked=1" else echo "walletLocked=0" fi # synced to chain syncedToChain=$(echo ${lndinfo} | jq -r '.synced_to_chain' | grep -c 'true') echo "syncedToChain=${syncedToChain}" # lnd scan progress scanTimestamp=$(echo ${lndinfo} | jq -r '.best_header_timestamp') if [ ${#scanTimestamp} -gt 0 ]; then echo "scanTimestamp=${scanTimestamp}" scanDate=$(date -d @${scanTimestamp}) echo "scanDate='${scanDate}'" # calculate LND scan progress by seconds since Genesisblock genesisTimestamp=1230940800 nowTimestamp=$(date +%s) totalSeconds=$(echo "${nowTimestamp}-${genesisTimestamp}" | bc) scannedSeconds=$(echo "${scanTimestamp}-${genesisTimestamp}" | bc) scanProgress=$(echo "scale=2; $scannedSeconds*100/$totalSeconds" | bc) echo "scanProgress=${scanProgress}" else echo "# was not able to parse 'best_header_timestamp' from: lncli getinfo" echo "scanTimestamp=-1" fi fi # output if lnd-RPC is ready echo "lndRPCReady=${lndRPCReady}" fi # check if online if problem with other stuff # info on scan run time endTime=$(date +%s) runTime=$(echo "${endTime}-${startTime}" | bc) echo "scriptRuntime=${runTime}"