chaintopology: load forwards, not backwards.

We used to load the new tip and work backwards until we joined up with
the previous tip.  That consumed quite a lot of memory if there were
many blocks.

Instead, just poll on blocknum+1, and grab it once that succeeds.  If
prev is different from what we expect (reorg), we free the current tip
and try again.

We could theoretically miss a reorg which is the same length (2 block
reorg with more work due to difficulty adjustment), but even if that
happened we'd catch up on the next block.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/chaintopology.c

@@ -17,7 +17,8 @@
 #include <common/utils.h>
 #include <inttypes.h>
 
-static void start_poll_chaintip(struct chain_topology *topo);
+/* Mutual recursion via timer. */
+static void try_extend_tip(struct chain_topology *topo);
 
 static void next_topology_timer(struct chain_topology *topo)
 {
@@ -27,7 +28,7 @@ static void next_topology_timer(struct chain_topology *topo)
 	}
 	/* This takes care of its own lifetime. */
 	notleak(new_reltimer(topo->timers, topo, topo->poll_time,
-			     start_poll_chaintip, topo));
+			     try_extend_tip, topo));
 }
 
 /* FIXME: Remove tx from block when peer done. */
@@ -54,23 +55,11 @@ static bool we_broadcast(const struct chain_topology *topo,
 	return false;
 }
 
-/* Fills in prev, height, mediantime. */
+static void filter_block_txs(struct chain_topology *topo, struct block *b)
-static void connect_block(struct chain_topology *topo,
-			  struct block *prev,
-			  struct block *b)
 {
 	size_t i;
 	u64 satoshi_owned;
 
-	assert(b->height == -1);
-	assert(b->prev == NULL);
-	assert(prev->next == b);
-
-	b->prev = prev;
-	b->height = b->prev->height + 1;
-
-	block_map_add(&topo->block_map, b);
-
 	/* Now we see if any of those txs are interesting. */
 	for (i = 0; i < tal_count(b->full_txs); i++) {
 		const struct bitcoin_tx *tx = b->full_txs[i];
@@ -102,9 +91,6 @@ static void connect_block(struct chain_topology *topo,
 			add_tx_to_block(b, tx, i);
 	}
 	b->full_txs = tal_free(b->full_txs);
-
-	/* Tell lightningd about new block. */
-	notify_new_block(topo->bitcoind->ld, b->height);
 }
 
 static const struct bitcoin_tx *tx_in_block(const struct block *b,
@@ -285,23 +271,6 @@ void broadcast_tx(struct chain_topology *topo,
 	bitcoind_sendrawtx(topo->bitcoind, otx->hextx, broadcast_done, otx);
 }
 
-static void free_blocks(struct chain_topology *topo, struct block *b)
-{
-	struct block *next;
-
-	while (b) {
-		size_t i, n = tal_count(b->txs);
-
-		/* Notify that txs are kicked out. */
-		for (i = 0; i < n; i++)
-			txwatch_fire(topo, b->txs[i], 0);
-
-		next = b->next;
-		tal_free(b);
-		b = next;
-	}
-}
-
 static const char *feerate_name(enum feerate feerate)
 {
 	return feerate == FEERATE_IMMEDIATE ? "Immediate"
@@ -367,21 +336,21 @@ static void next_updatefee_timer(struct chain_topology *topo)
 			     start_fee_estimate, topo));
 }
 
-/* B is the new chain (linked by ->next); update topology */
+static void add_tip(struct chain_topology *topo, struct block *b)
-static void topology_changed(struct chain_topology *topo,
-			     struct block *prev,
-			     struct block *b)
 {
-	/* Eliminate any old chain. */
+	/* Only keep the transactions we care about. */
-	if (prev->next)
+	filter_block_txs(topo, b);
-		free_blocks(topo, prev->next);
 
-	prev->next = b;
+	block_map_add(&topo->block_map, b);
-	do {
+
-		connect_block(topo, prev, b);
+	/* Attach to tip; b is now the tip. */
-		topo->tip = prev = b;
+	assert(b->height == topo->tip->height + 1);
-		b = b->next;
+	b->prev = topo->tip;
-	} while (b);
+	topo->tip->next = b;
+	topo->tip = b;
+
+	/* Tell lightningd about new block. */
+	notify_new_block(topo->bitcoind->ld, b->height);
 
 	/* Tell watch code to re-evaluate all txs. */
 	watch_topology_changed(topo);
@@ -392,7 +361,7 @@ static void topology_changed(struct chain_topology *topo,
 
 static struct block *new_block(struct chain_topology *topo,
 			       struct bitcoin_block *blk,
-			       struct block *next)
+			       unsigned int height)
 {
 	struct block *b = tal(topo, struct block);
 
@@ -400,12 +369,11 @@ static struct block *new_block(struct chain_topology *topo,
 	log_debug(topo->log, "Adding block %s",
 		  type_to_string(ltmp, struct bitcoin_blkid, &b->blkid));
 	assert(!block_map_get(&topo->block_map, &b->blkid));
-	b->next = next;
+	b->next = NULL;
+	b->prev = NULL;
 	b->topo = topo;
 
-	/* We fill these out in topology_changed */
+	b->height = height;
-	b->height = -1;
-	b->prev = NULL;
 
 	b->hdr = blk->hdr;
 
@@ -416,80 +384,66 @@ static struct block *new_block(struct chain_topology *topo,
 	return b;
 }
 
-static void add_block(struct bitcoind *bitcoind,
+static void remove_tip(struct chain_topology *topo)
-		      struct chain_topology *topo,
-		      struct bitcoin_block *blk,
-		      struct block *next);
-
-static void gather_previous_blocks(struct bitcoind *bitcoind,
-				   struct bitcoin_block *blk,
-				   struct block *next)
 {
-	add_block(bitcoind, next->topo, blk, next);
+	struct block *b = topo->tip;
-}
+	size_t i, n = tal_count(b->txs);
-
-static void add_block(struct bitcoind *bitcoind,
-		      struct chain_topology *topo,
-		      struct bitcoin_block *blk,
-		      struct block *next)
-{
-	struct block *b, *prev;
 
-	b = new_block(topo, blk, next);
+	/* Move tip back one. */
+	topo->tip = b->prev;
+	if (!topo->tip)
+		fatal("Initial block %u (%s) reorganized out!",
+		      b->height,
+		      type_to_string(ltmp, struct bitcoin_blkid, &b->blkid));
 
-	/* Recurse if we need prev. */
+	/* Notify that txs are kicked out. */
-	prev = block_map_get(&topo->block_map, &blk->hdr.prev_hash);
+	for (i = 0; i < n; i++)
-	if (!prev) {
+		txwatch_fire(topo, b->txs[i], 0);
-		bitcoind_getrawblock(bitcoind, &blk->hdr.prev_hash,
-				     gather_previous_blocks, b);
-		return;
-	}
 
-	/* All done. */
+	tal_free(b);
-	topology_changed(topo, prev, b);
-	next_topology_timer(topo);
 }
 
-static void rawblock_tip(struct bitcoind *bitcoind,
+static void have_new_block(struct bitcoind *bitcoind,
 			   struct bitcoin_block *blk,
 			   struct chain_topology *topo)
 {
-	add_block(bitcoind, topo, blk, NULL);
+	/* Unexpected precessor?  Free predecessor, refetch it. */
+	if (!structeq(&topo->tip->blkid, &blk->hdr.prev_hash))
+		remove_tip(topo);
+	else
+		add_tip(topo, new_block(topo, blk, topo->tip->height + 1));
+
+	/* Try for next one. */
+	try_extend_tip(topo);
 }
 
-static void check_chaintip(struct bitcoind *bitcoind,
+static void get_new_block(struct bitcoind *bitcoind,
-			   const struct bitcoin_blkid *tipid,
+			  const struct bitcoin_blkid *blkid,
 			  struct chain_topology *topo)
 {
-	/* 0 is the main tip. */
+	if (!blkid) {
-	if (!structeq(tipid, &topo->tip->blkid))
+		/* No such block, poll. */
-		bitcoind_getrawblock(bitcoind, tipid, rawblock_tip, topo);
-	else
-		/* Next! */
 		next_topology_timer(topo);
+		return;
+	}
+	bitcoind_getrawblock(bitcoind, blkid, have_new_block, topo);
 }
 
-static void start_poll_chaintip(struct chain_topology *topo)
+static void try_extend_tip(struct chain_topology *topo)
 {
-	if (!list_empty(&topo->bitcoind->pending)) {
+	bitcoind_getblockhash(topo->bitcoind, topo->tip->height + 1,
-		log_unusual(topo->log,
+			      get_new_block, topo);
-			    "Delaying start poll: commands in progress");
-		next_topology_timer(topo);
-	} else
-		bitcoind_get_chaintip(topo->bitcoind, check_chaintip, topo);
 }
 
 static void init_topo(struct bitcoind *bitcoind,
 		      struct bitcoin_block *blk,
 		      struct chain_topology *topo)
 {
-	topo->root = new_block(topo, blk, NULL);
+	topo->root = new_block(topo, blk, topo->first_blocknum);
-	topo->root->height = topo->first_blocknum;
 	block_map_add(&topo->block_map, topo->root);
 	topo->tip = topo->root;
 
-	/* Now grab chaintip immediately. */
+	try_extend_tip(topo);
-	bitcoind_get_chaintip(bitcoind, check_chaintip, topo);
 }
 
 static void get_init_block(struct bitcoind *bitcoind,