From 5527b545b6bd0a6b271f985ee90a6cee2edd8c99 Mon Sep 17 00:00:00 2001 From: Alexis Hernandez Date: Wed, 27 Feb 2019 21:51:00 -0700 Subject: [PATCH] server: Add lite-version config Enabling the lite-version allows to prune block details to allow syncing fast until what the light wallet needs. --- .../xsn/explorer/config/ExplorerConfig.scala | 27 ++++++++++++++----- ...AddressTransactionDetailsPostgresDAO.scala | 14 +++++++--- .../dao/TransactionInputPostgresDAO.scala | 11 +++++--- .../dao/TransactionOutputPostgresDAO.scala | 14 +++++++--- .../anorm/dao/TransactionPostgresDAO.scala | 8 +++++- .../xsn/explorer/modules/ConfigModule.scala | 2 +- .../services/LedgerSynchronizerService.scala | 11 +++++++- server/conf/application.conf | 17 +++++++++++- .../com/xsn/explorer/helpers/Config.scala | 13 +++++++++ .../explorer/helpers/DataHandlerObjects.scala | 9 ++++--- .../LedgerSynchronizerServiceSpec.scala | 1 + .../services/XSNServiceRPCImplSpec.scala | 2 ++ 12 files changed, 105 insertions(+), 24 deletions(-) create mode 100644 server/test/com/xsn/explorer/helpers/Config.scala diff --git a/server/app/com/xsn/explorer/config/ExplorerConfig.scala b/server/app/com/xsn/explorer/config/ExplorerConfig.scala index 446e098..69b83fa 100644 --- a/server/app/com/xsn/explorer/config/ExplorerConfig.scala +++ b/server/app/com/xsn/explorer/config/ExplorerConfig.scala @@ -7,15 +7,28 @@ import play.api.Configuration trait ExplorerConfig { def genesisBlock: Blockhash + + def liteVersionConfig: ExplorerConfig.LiteVersionConfig } -class PlayExplorerConfig @Inject() (config: Configuration) extends ExplorerConfig { +object ExplorerConfig { - override val genesisBlock: Blockhash = { - Blockhash - .from(config.get[String]("explorer.genesisBlock")) - .getOrElse(throw new RuntimeException("The given genesisBlock is incorrect")) - } + case class LiteVersionConfig(enabled: Boolean, syncTransactionsFromBlock: Int) -} + class Play @Inject() (config: Configuration) extends ExplorerConfig { + + override val genesisBlock: Blockhash = { + Blockhash + .from(config.get[String]("explorer.genesisBlock")) + .getOrElse(throw new RuntimeException("The given genesisBlock is incorrect")) + } + override def liteVersionConfig: LiteVersionConfig = { + val inner = config.get[Configuration]("explorer.liteVersion") + val enabled = inner.get[Boolean]("enabled") + val syncTransactionsFromBlock = inner.get[Int]("syncTransactionsFromBlock") + + LiteVersionConfig(enabled, syncTransactionsFromBlock) + } + } +} diff --git a/server/app/com/xsn/explorer/data/anorm/dao/AddressTransactionDetailsPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/AddressTransactionDetailsPostgresDAO.scala index 8907ec5..5739d6c 100644 --- a/server/app/com/xsn/explorer/data/anorm/dao/AddressTransactionDetailsPostgresDAO.scala +++ b/server/app/com/xsn/explorer/data/anorm/dao/AddressTransactionDetailsPostgresDAO.scala @@ -3,11 +3,16 @@ package com.xsn.explorer.data.anorm.dao import java.sql.Connection import anorm._ +import com.xsn.explorer.config.ExplorerConfig import com.xsn.explorer.data.anorm.parsers.TransactionParsers._ import com.xsn.explorer.models.persisted.{AddressTransactionDetails, Transaction} import com.xsn.explorer.models.values.TransactionId +import javax.inject.Inject +import org.slf4j.LoggerFactory -class AddressTransactionDetailsPostgresDAO { +class AddressTransactionDetailsPostgresDAO @Inject() (explorerConfig: ExplorerConfig) { + + private val logger = LoggerFactory.getLogger(this.getClass) def batchInsertDetails(transaction: Transaction.HasIO)(implicit conn: Connection): Option[Unit] = { val outputAddressValueList = for { @@ -66,9 +71,12 @@ class AddressTransactionDetailsPostgresDAO { params.tail: _* ) - val success = batch.execute().forall(_ == 1) + val result = batch.execute() + val success = result.forall(_ == 1) + + if (success || + explorerConfig.liteVersionConfig.enabled) { - if (success) { Some(()) } else { None diff --git a/server/app/com/xsn/explorer/data/anorm/dao/TransactionInputPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/TransactionInputPostgresDAO.scala index b2a28e1..da674bb 100644 --- a/server/app/com/xsn/explorer/data/anorm/dao/TransactionInputPostgresDAO.scala +++ b/server/app/com/xsn/explorer/data/anorm/dao/TransactionInputPostgresDAO.scala @@ -3,12 +3,14 @@ package com.xsn.explorer.data.anorm.dao import java.sql.Connection import anorm._ +import com.xsn.explorer.config.ExplorerConfig import com.xsn.explorer.data.anorm.parsers.TransactionParsers._ import com.xsn.explorer.models.persisted.Transaction import com.xsn.explorer.models.values.{Address, TransactionId} +import javax.inject.Inject import org.slf4j.LoggerFactory -class TransactionInputPostgresDAO { +class TransactionInputPostgresDAO @Inject() (explorerConfig: ExplorerConfig) { private val logger = LoggerFactory.getLogger(this.getClass) @@ -41,8 +43,11 @@ class TransactionInputPostgresDAO { params.tail: _* ) - val success = batch.execute().forall(_ == 1) - if (success) { + val result = batch.execute() + val success = result.forall(_ == 1) + if (success || + explorerConfig.liteVersionConfig.enabled) { + Some(inputs) } else { None diff --git a/server/app/com/xsn/explorer/data/anorm/dao/TransactionOutputPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/TransactionOutputPostgresDAO.scala index fb4801a..a10076d 100644 --- a/server/app/com/xsn/explorer/data/anorm/dao/TransactionOutputPostgresDAO.scala +++ b/server/app/com/xsn/explorer/data/anorm/dao/TransactionOutputPostgresDAO.scala @@ -3,12 +3,14 @@ package com.xsn.explorer.data.anorm.dao import java.sql.Connection import anorm._ +import com.xsn.explorer.config.ExplorerConfig import com.xsn.explorer.data.anorm.parsers.TransactionParsers._ import com.xsn.explorer.models.persisted.Transaction import com.xsn.explorer.models.values.{Address, TransactionId} +import javax.inject.Inject import org.slf4j.LoggerFactory -class TransactionOutputPostgresDAO { +class TransactionOutputPostgresDAO @Inject() (explorerConfig: ExplorerConfig) { private val logger = LoggerFactory.getLogger(this.getClass) @@ -68,9 +70,11 @@ class TransactionOutputPostgresDAO { ) val success = batch.execute().forall(_ == 1) - if (success) { + if (success || + explorerConfig.liteVersionConfig.enabled) { + Some(outputs) - } else { + } else{ None } } @@ -145,7 +149,9 @@ class TransactionOutputPostgresDAO { """.stripMargin ).executeUpdate() - if (result == inputs.size) { + if (result == inputs.size || + explorerConfig.liteVersionConfig.enabled) { + Option(()) } else { None diff --git a/server/app/com/xsn/explorer/data/anorm/dao/TransactionPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/TransactionPostgresDAO.scala index 6d3dcac..bbf3f6b 100644 --- a/server/app/com/xsn/explorer/data/anorm/dao/TransactionPostgresDAO.scala +++ b/server/app/com/xsn/explorer/data/anorm/dao/TransactionPostgresDAO.scala @@ -6,6 +6,7 @@ import anorm._ import com.alexitc.playsonify.models.ordering.{FieldOrdering, OrderingCondition} import com.alexitc.playsonify.models.pagination.{Count, Limit, PaginatedQuery} import com.alexitc.playsonify.sql.FieldOrderingSQLInterpreter +import com.xsn.explorer.config.ExplorerConfig import com.xsn.explorer.data.anorm.parsers.TransactionParsers._ import com.xsn.explorer.models._ import com.xsn.explorer.models.fields.TransactionField @@ -15,6 +16,7 @@ import javax.inject.Inject import org.slf4j.LoggerFactory class TransactionPostgresDAO @Inject() ( + explorerConfig: ExplorerConfig, transactionInputDAO: TransactionInputPostgresDAO, transactionOutputDAO: TransactionOutputPostgresDAO, tposContractDAO: TPoSContractDAO, @@ -65,7 +67,11 @@ class TransactionPostgresDAO @Inject() ( private def spend(transactions: List[Transaction.HasIO])(implicit conn: Connection): Unit = { val spendResult = transactions.map { tx => transactionOutputDAO.batchSpend(tx.id, tx.inputs) } - assert(spendResult.forall(_.isDefined), "Spending inputs batch failed") + if (explorerConfig.liteVersionConfig.enabled) { + () + } else { + assert(spendResult.forall(_.isDefined), "Spending inputs batch failed") + } } private def closeContracts(transactions: List[Transaction.HasIO])(implicit conn: Connection): Unit = { diff --git a/server/app/com/xsn/explorer/modules/ConfigModule.scala b/server/app/com/xsn/explorer/modules/ConfigModule.scala index f790b79..2ef3b79 100644 --- a/server/app/com/xsn/explorer/modules/ConfigModule.scala +++ b/server/app/com/xsn/explorer/modules/ConfigModule.scala @@ -8,6 +8,6 @@ class ConfigModule extends AbstractModule { override def configure(): Unit = { bind(classOf[RPCConfig]).to(classOf[PlayRPCConfig]) bind(classOf[LedgerSynchronizerConfig]).to(classOf[LedgerSynchronizerPlayConfig]) - bind(classOf[ExplorerConfig]).to(classOf[PlayExplorerConfig]) + bind(classOf[ExplorerConfig]).to(classOf[ExplorerConfig.Play]) } } diff --git a/server/app/com/xsn/explorer/services/LedgerSynchronizerService.scala b/server/app/com/xsn/explorer/services/LedgerSynchronizerService.scala index 2de9d5e..333135a 100644 --- a/server/app/com/xsn/explorer/services/LedgerSynchronizerService.scala +++ b/server/app/com/xsn/explorer/services/LedgerSynchronizerService.scala @@ -2,6 +2,7 @@ package com.xsn.explorer.services import com.alexitc.playsonify.core.FutureOr.Implicits.{FutureOps, OptionOps} import com.alexitc.playsonify.core.{ApplicationResult, FutureApplicationResult} +import com.xsn.explorer.config.ExplorerConfig import com.xsn.explorer.data.async.{BlockFutureDataHandler, LedgerFutureDataHandler} import com.xsn.explorer.errors.BlockNotFoundError import com.xsn.explorer.models._ @@ -16,6 +17,7 @@ import org.slf4j.LoggerFactory import scala.concurrent.{ExecutionContext, Future} class LedgerSynchronizerService @Inject() ( + explorerConfig: ExplorerConfig, xsnService: XSNService, blockService: BlockService, transactionCollectorService: TransactionCollectorService, @@ -172,7 +174,14 @@ class LedgerSynchronizerService @Inject() ( val result = for { rpcBlock <- xsnService.getBlock(blockhash).toFutureOr } yield { - rpcBlock + if (explorerConfig.liteVersionConfig.enabled && + rpcBlock.height.int < explorerConfig.liteVersionConfig.syncTransactionsFromBlock) { + + // lite version, ignore transactions + rpcBlock.copy(transactions = List.empty) + } else { + rpcBlock + } } result.toFuture diff --git a/server/conf/application.conf b/server/conf/application.conf index f901f3e..2c3e66f 100644 --- a/server/conf/application.conf +++ b/server/conf/application.conf @@ -42,7 +42,7 @@ rpc { synchronizer { enabled = true initialDelay = "10 seconds" - interval = "1 minute" + interval = "1 minutes" enabled = ${?XSN_SYNCHRONIZER_ENABLED} } @@ -110,4 +110,19 @@ externalService.dispatcher { explorer { genesisBlock = "00000c822abdbb23e28f79a49d29b41429737c6c7e15df40d1b1f1b35907ae34" genesisBlock = ${?EXPLORER_GENESIS_BLOCK} + + liteVersion { + # Enabling the lite explorer, starts syncing the block details only, the transactions and + # its related details are synced only on blocks having height >= syncTransactionsFromBlock + # + # This is useful to support our light wallet, assuming that all its addresses are new (hence, old details aren't important). + # + # This also implies that some consistency checks are removed, you are required to drop some foreign keys manually. + # + # Updating these values on an existing explorer is likely to need to rebuilt the database from scratch. + enabled = false + enabled = ${?EXPLORER_LITE_VERSION_ENABLED} + + syncTransactionsFromBlock = 0 + } } diff --git a/server/test/com/xsn/explorer/helpers/Config.scala b/server/test/com/xsn/explorer/helpers/Config.scala new file mode 100644 index 0000000..a7be60b --- /dev/null +++ b/server/test/com/xsn/explorer/helpers/Config.scala @@ -0,0 +1,13 @@ +package com.xsn.explorer.helpers + +import com.xsn.explorer.config.ExplorerConfig +import com.xsn.explorer.models.values.Blockhash + +object Config { + val explorerConfig = new ExplorerConfig { + + override def genesisBlock: Blockhash = Blockhash.from("00000c822abdbb23e28f79a49d29b41429737c6c7e15df40d1b1f1b35907ae34").get + + override def liteVersionConfig: ExplorerConfig.LiteVersionConfig = ExplorerConfig.LiteVersionConfig(enabled = false, 0) + } +} diff --git a/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala b/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala index e648940..f710ea3 100644 --- a/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala +++ b/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala @@ -7,12 +7,15 @@ import play.api.db.Database trait DataHandlerObjects { + import Config.explorerConfig + lazy val fieldOrderingSQLInterpreter = new FieldOrderingSQLInterpreter - lazy val transactionInputDAO = new TransactionInputPostgresDAO - lazy val transactionOutputDAO = new TransactionOutputPostgresDAO - lazy val addressTransactionDetailsDAO = new AddressTransactionDetailsPostgresDAO + lazy val transactionInputDAO = new TransactionInputPostgresDAO(explorerConfig) + lazy val transactionOutputDAO = new TransactionOutputPostgresDAO(explorerConfig) + lazy val addressTransactionDetailsDAO = new AddressTransactionDetailsPostgresDAO(explorerConfig) lazy val tposContractDAO = new TPoSContractDAO lazy val transactionPostgresDAO = new TransactionPostgresDAO( + explorerConfig, transactionInputDAO, transactionOutputDAO, tposContractDAO, diff --git a/server/test/com/xsn/explorer/services/LedgerSynchronizerServiceSpec.scala b/server/test/com/xsn/explorer/services/LedgerSynchronizerServiceSpec.scala index bf5fa7a..015e087 100644 --- a/server/test/com/xsn/explorer/services/LedgerSynchronizerServiceSpec.scala +++ b/server/test/com/xsn/explorer/services/LedgerSynchronizerServiceSpec.scala @@ -227,6 +227,7 @@ class LedgerSynchronizerServiceSpec extends PostgresDataHandlerSpec with BeforeA new TransactionFutureDataHandler(transactionDataHandler)(Executors.databaseEC) ) new LedgerSynchronizerService( + Config.explorerConfig, xsnService, blockService, transactionCollectorService, diff --git a/server/test/com/xsn/explorer/services/XSNServiceRPCImplSpec.scala b/server/test/com/xsn/explorer/services/XSNServiceRPCImplSpec.scala index 53b5002..e930b2d 100644 --- a/server/test/com/xsn/explorer/services/XSNServiceRPCImplSpec.scala +++ b/server/test/com/xsn/explorer/services/XSNServiceRPCImplSpec.scala @@ -33,6 +33,8 @@ class XSNServiceRPCImplSpec extends WordSpec { val explorerConfig = new ExplorerConfig { override def genesisBlock: Blockhash = Blockhash.from("00000c822abdbb23e28f79a49d29b41429737c6c7e15df40d1b1f1b35907ae34").get + + override def liteVersionConfig: ExplorerConfig.LiteVersionConfig = ??? } val request = mock[WSRequest]