From 52ae52e3c47615180d1b0ddfe4bd3257fdd7dd67 Mon Sep 17 00:00:00 2001 From: Alexis Hernandez Date: Mon, 18 Mar 2019 23:21:25 -0700 Subject: [PATCH] server: Store the block filter while synchronizing the ledger --- .../anorm/LedgerPostgresDataHandler.scala | 16 +++++-- .../anorm/dao/BlockFilterPostgresDAO.scala | 43 +++++++++++++++++++ .../anorm/parsers/BlockFilterParsers.scala | 26 +++++++++++ .../data/common/PostgresDataHandlerSpec.scala | 2 +- .../explorer/helpers/DataHandlerObjects.scala | 2 + 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala create mode 100644 server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala diff --git a/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala b/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala index 0586f7b..7276377 100644 --- a/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala +++ b/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala @@ -5,8 +5,9 @@ import java.sql.Connection import com.alexitc.playsonify.core.ApplicationResult import com.alexitc.playsonify.models.ApplicationError import com.xsn.explorer.data.LedgerBlockingDataHandler -import com.xsn.explorer.data.anorm.dao.{AggregatedAmountPostgresDAO, BalancePostgresDAO, BlockPostgresDAO, TransactionPostgresDAO} +import com.xsn.explorer.data.anorm.dao._ import com.xsn.explorer.errors.{PostgresForeignKeyViolationError, PreviousBlockMissingError, RepeatedBlockHeightError} +import com.xsn.explorer.gcs.{GolombCodedSet, GolombEncoding} import com.xsn.explorer.models.persisted.{Balance, Block, Transaction} import com.xsn.explorer.models.values.Address import com.xsn.explorer.util.Extensions.ListOptionExt @@ -18,6 +19,7 @@ import play.api.db.Database class LedgerPostgresDataHandler @Inject() ( override val database: Database, blockPostgresDAO: BlockPostgresDAO, + blockFilterPostgresDAO: BlockFilterPostgresDAO, transactionPostgresDAO: TransactionPostgresDAO, balancePostgresDAO: BalancePostgresDAO, aggregatedAmountPostgresDAO: AggregatedAmountPostgresDAO) @@ -33,9 +35,11 @@ class LedgerPostgresDataHandler @Inject() ( override def push( block: Block.HasTransactions): ApplicationResult[Unit] = { + // the filter is computed outside the transaction to avoid unnecessary locking + val filter = GolombEncoding.encode(block) val result = withTransaction { implicit conn => val result = for { - _ <- upsertBlockCascade(block.asTip) + _ <- upsertBlockCascade(block.asTip, filter) } yield () result @@ -66,11 +70,16 @@ class LedgerPostgresDataHandler @Inject() ( .getOrElse(throw new RuntimeException("Unable to pop block")) } - private def upsertBlockCascade(block: Block.HasTransactions)(implicit conn: Connection): Option[Unit] = { + private def upsertBlockCascade( + block: Block.HasTransactions, + filter: Option[GolombCodedSet])( + implicit conn: Connection): Option[Unit] = { + val result = for { // block _ <- deleteBlockCascade(block.block).orElse(Some(())) _ <- blockPostgresDAO.insert(block.block) + _ = filter.foreach { f => blockFilterPostgresDAO.insert(block.hash, f) } // batch insert _ <- transactionPostgresDAO.insert(block.transactions) @@ -96,6 +105,7 @@ class LedgerPostgresDataHandler @Inject() ( private def deleteBlockCascade(block: Block)(implicit conn: Connection): Option[Unit] = { // transactions val deletedTransactions = transactionPostgresDAO.deleteBy(block.hash) + val _ = blockFilterPostgresDAO.delete(block.hash) for { // block _ <- blockPostgresDAO.delete(block.hash) diff --git a/server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala new file mode 100644 index 0000000..d513906 --- /dev/null +++ b/server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala @@ -0,0 +1,43 @@ +package com.xsn.explorer.data.anorm.dao + +import java.sql.Connection + +import anorm._ +import com.xsn.explorer.data.anorm.parsers.BlockFilterParsers +import com.xsn.explorer.gcs.GolombCodedSet +import com.xsn.explorer.models.values.Blockhash + +class BlockFilterPostgresDAO { + + import BlockFilterParsers._ + + def insert(blockhash: Blockhash, filter: GolombCodedSet)(implicit conn: Connection): GolombCodedSet = { + SQL( + """ + |INSERT INTO block_address_gcs + | (blockhash, m, n, p, hex) + |VALUES + | ({blockhash}, {m}, {n}, {p}, {hex}) + |RETURNING blockhash, m, n, p, hex + """.stripMargin + ).on( + 'blockhash -> blockhash.string, + 'm -> filter.m, + 'n -> filter.n, + 'p -> filter.p, + 'hex -> filter.hex.string + ).as(parseFilter.single) + } + + def delete(blockhash: Blockhash)(implicit conn: Connection): Option[GolombCodedSet] = { + SQL( + """ + |DELETE FROM block_address_gcs + |WHERE blockhash = {blockhash} + |RETURNING blockhash, m, n, p, hex + """.stripMargin + ).on( + 'blockhash -> blockhash.string + ).as(parseFilter.singleOpt) + } +} diff --git a/server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala b/server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala new file mode 100644 index 0000000..bc31566 --- /dev/null +++ b/server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala @@ -0,0 +1,26 @@ +package com.xsn.explorer.data.anorm.parsers + +import anorm.SqlParser._ +import anorm._ +import com.xsn.explorer.gcs.GolombCodedSet +import com.xsn.explorer.models.values.HexString + +object BlockFilterParsers { + + val parseP = int("p") + val parseM = int("m") + val parseN = int("n") + val parseHex = str("hex") + .map(HexString.from) + .map { _.getOrElse(throw new RuntimeException("corrupted hex")) } + + val parseFilter = (parseN ~ parseM ~ parseP ~ parseHex).map { + case n ~ m ~ p ~ hex => + new GolombCodedSet( + n = n, + m = m, + p = p, + hex = hex + ) + } +} diff --git a/server/test/com/xsn/explorer/data/common/PostgresDataHandlerSpec.scala b/server/test/com/xsn/explorer/data/common/PostgresDataHandlerSpec.scala index 0c908bf..f83828e 100644 --- a/server/test/com/xsn/explorer/data/common/PostgresDataHandlerSpec.scala +++ b/server/test/com/xsn/explorer/data/common/PostgresDataHandlerSpec.scala @@ -64,10 +64,10 @@ trait PostgresDataHandlerSpec _root_.anorm.SQL("""DELETE FROM transaction_outputs""").execute() _root_.anorm.SQL("""DELETE FROM address_transaction_details""").execute() _root_.anorm.SQL("""DELETE FROM transactions""").execute() + _root_.anorm.SQL("""DELETE FROM block_address_gcs""").execute() _root_.anorm.SQL("""DELETE FROM blocks""").execute() _root_.anorm.SQL("""DELETE FROM balances""").execute() _root_.anorm.SQL("""DELETE FROM hidden_addresses""").execute() } } - } diff --git a/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala b/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala index ac3c8bd..af72693 100644 --- a/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala +++ b/server/test/com/xsn/explorer/helpers/DataHandlerObjects.scala @@ -17,6 +17,7 @@ trait DataHandlerObjects { addressTransactionDetailsDAO, fieldOrderingSQLInterpreter) lazy val blockPostgresDAO = new BlockPostgresDAO(fieldOrderingSQLInterpreter) + lazy val blockFilterPostgresDAO = new BlockFilterPostgresDAO lazy val balancePostgresDAO = new BalancePostgresDAO(fieldOrderingSQLInterpreter) lazy val aggregatedAmountPostgresDAO = new AggregatedAmountPostgresDAO @@ -24,6 +25,7 @@ trait DataHandlerObjects { new LedgerPostgresDataHandler( database, blockPostgresDAO, + blockFilterPostgresDAO, transactionPostgresDAO, balancePostgresDAO, aggregatedAmountPostgresDAO)