daemon: implement anchor watch timeout.

We abort when this happens, but still worth testing.

This involves a refactor so we can allocate watches off a specific context,
for easy freeing when they're no longer wanted.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -205,7 +205,7 @@ $(TEST_CLI_PROGRAMS:=.o): $(TEST_CLI_HEADERS)
 # These don't work in parallel, so we open-code them
 test-cli-tests: $(TEST_CLI_PROGRAMS) daemon-all
 	cd test-cli; scripts/shutdown.sh 2>/dev/null || true
-	set -e; for args in ""; do daemon/test/test.sh; done
+	set -e; for arg in "" "--timeout-anchor"; do daemon/test/test.sh $$arg; done
 	set -e; cd test-cli; for args in "" --steal --unilateral --htlc-onchain; do scripts/setup.sh && scripts/test.sh $$args && scripts/shutdown.sh; done
 
 test-onion: test/test_onion test/onion_key

daemon/peer.c

@@ -343,6 +343,7 @@ static struct peer *new_peer(struct lightningd_state *dstate,
 	peer->close_tx = NULL;
 	peer->cstate = NULL;
 	peer->close_watch_timeout = NULL;
+	peer->anchor.watches = NULL;
 
 	/* If we free peer, conn should be closed, but can't be freed
 	 * immediately so don't make peer a parent. */
@@ -545,11 +546,15 @@ const struct json_command connect_command = {
 };
 
 struct anchor_watch {
+	struct peer *peer;
 	enum state_input depthok;
 	enum state_input timeout;
 	enum state_input unspent;
 	enum state_input theyspent;
 	enum state_input otherspent;
+
+	/* If timeout != INPUT_NONE, this is the timer. */
+	struct oneshot *timer;
 };
 
 static void anchor_depthchange(struct peer *peer, int depth,
@@ -561,6 +566,8 @@ static void anchor_depthchange(struct peer *peer, int depth,
 		if (depth >= (int)peer->us.mindepth) {
 			enum state_input in = w->depthok;
 			w->depthok = INPUT_NONE;
+			/* We don't need the timeout timer any more. */
+			w->timer = tal_free(w->timer);
 			state_event(peer, in, NULL);
 		}
 	} else {
@@ -646,11 +653,20 @@ static void anchor_spent(struct peer *peer,
 	if (txmatch(tx, peer->them.commit))
 		state_event(peer, w->theyspent, &idata);
 	else if (is_mutual_close(tx, peer->close_tx))
-		add_close_tx_watch(peer, tx, close_depth_cb);
+		add_close_tx_watch(peer, peer, tx, close_depth_cb);
 	else
 		state_event(peer, w->otherspent, &idata);
 }
 
+static void anchor_timeout(struct anchor_watch *w)
+{
+	assert(w == w->peer->anchor.watches);
+	state_event(w->peer, w->timeout, NULL);
+
+	/* Freeing this gets rid of the other watches, and timer, too. */
+	w->peer->anchor.watches = tal_free(w);
+}
+
 void peer_watch_anchor(struct peer *peer,
 		       enum state_input depthok,
 		       enum state_input timeout,
@@ -658,27 +674,57 @@ void peer_watch_anchor(struct peer *peer,
 		       enum state_input theyspent,
 		       enum state_input otherspent)
 {
-	struct anchor_watch *w = tal(peer, struct anchor_watch);
+	struct anchor_watch *w;
+
+	w = peer->anchor.watches = tal(peer, struct anchor_watch);
 
+	w->peer = peer;
 	w->depthok = depthok;
 	w->timeout = timeout;
 	w->unspent = unspent;
 	w->theyspent = theyspent;
 	w->otherspent = otherspent;
 
-	add_anchor_watch(peer, &peer->anchor.txid, peer->anchor.index,
+	add_anchor_watch(w, peer, &peer->anchor.txid, peer->anchor.index,
 			 anchor_depthchange,
 			 anchor_spent,
 			 w);
 
-	/* FIXME: add timeout */
+	/* For anchor timeout, expect 20 minutes per block, +2 hours.
+	 *
+	 * Probability(no block in time N) = e^(-N/600).
+	 * Thus for 1 block, P = e^(-(7200+1*1200)/600) = 0.83 in a million.
+	 *
+	 * Glenn Willen says, if we want to know how many 10-minute intervals for
+	 * a 1 in a million chance of spurious failure for N blocks, put
+	 * this into http://www.wolframalpha.com:
+	 *
+	 *   e^(-x) * sum x^i / fact(i), i=0 to N < 1/1000000
+	 * 
+	 * N=20: 51
+	 * N=10: 35
+	 * N=8:  31
+	 * N=6:  28
+	 * N=4:  24
+	 * N=3:  22
+	 * N=2:  20
+	 *
+	 * So, our formula of 12 + N*2 holds for N <= 20 at least.
+	 */
+	if (w->timeout != INPUT_NONE) {
+		w->timer = oneshot_timeout(peer->dstate, w,
+					   7200 + 20*peer->us.mindepth,
+					   anchor_timeout, w);
+	} else
+		w->timer = NULL;
 }
 
 void peer_unwatch_anchor_depth(struct peer *peer,
 			       enum state_input depthok,
 			       enum state_input timeout)
 {
-	FIXME_STUB(peer);
+	assert(peer->anchor.watches);
+	peer->anchor.watches = tal_free(peer->anchor.watches);
 }
 
 void peer_watch_delayed(struct peer *peer,

daemon/peer.h

@@ -94,6 +94,7 @@ struct peer {
 		u8 *redeemscript;
 		/* If we created it, we keep entire tx. */
 		const struct bitcoin_tx *tx;
+		struct anchor_watch *watches;
 	} anchor;
 
 	/* Their signature for our current commit sig. */
@@ -104,6 +105,7 @@ struct peer {
 	/* Number of HTLC updates (== number of previous commit txs) */
 	u64 num_htlcs;
 
+	/* FIXME: Group closing fields together in anon struct. */
 	/* Closing tx and signature once we've generated it */
 	struct bitcoin_tx *close_tx;
 	struct bitcoin_signature our_close_sig;

daemon/test/test.sh

@@ -18,12 +18,15 @@ if [ x"$1" = x"--valgrind" ]; then
     PREFIX2="valgrind --vgdb-error=1"
     REDIR1="/dev/tty"
     REDIR2="/dev/tty"
+    shift
 elif [ x"$1" = x"--gdb1" ]; then
     PREFIX1="gdb --args -ex run"
     REDIR1="/dev/tty"
+    shift
 elif [ x"$1" = x"--gdb2" ]; then
     PREFIX2="gdb --args -ex run"
     REDIR2="/dev/tty"
+    shift
 fi
 
 LCLI1="../daemon/lightning-cli --lightning-dir=$DIR1"
@@ -50,7 +53,15 @@ check_status()
 	return 1
     fi
 }
-   
+
+all_ok()
+{
+    scripts/shutdown.sh
+
+    trap "rm -rf $DIR1 $DIR2" EXIT
+    exit 0
+}
+    
 trap "echo Results in $DIR1 and $DIR2" EXIT
 mkdir $DIR1 $DIR2
 $PREFIX1 ../daemon/lightningd --log-level=debug --bitcoind-poll=1 --min-expiry=900 --lightning-dir=$DIR1 > $REDIR1 &
@@ -87,6 +98,36 @@ sleep 2
 $LCLI1 getpeers | grep STATE_OPEN_WAITING_OURANCHOR
 $LCLI2 getpeers | grep STATE_OPEN_WAITING_THEIRANCHOR
 
+if [ "x$1" = x"--timeout-anchor" ]; then
+    # Timeout before anchor committed.
+    TIME=$((`date +%s` + 7200 + 3 * 1200 + 1))
+
+    # This will crash in a moment.
+    $LCLI1 dev-mocktime $TIME
+
+    # This will crash immediately
+    if $LCLI2 dev-mocktime $TIME >&2; then
+	echo Node2 did not crash >&2
+	exit 1
+    fi
+
+    sleep 1
+
+    # Check crash logs
+    if [ ! -f $DIR1/crash.log ]; then
+	echo Node1 did not crash >&2
+	exit 1
+    fi
+    if [ ! -f $DIR2/crash.log ]; then
+	echo Node2 did not crash >&2
+	exit 1
+    fi
+
+    fgrep 'Entered error state STATE_ERR_ANCHOR_TIMEOUT' $DIR2/crash.log
+    all_ok
+fi
+    
+
 # Now make it pass anchor.
 $CLI generate 3
 
@@ -158,8 +199,9 @@ $LCLI2 getpeers | tr -s '\012\011 ' ' ' | fgrep '"STATE_CLOSE_WAIT_CLOSE"'
 
 # Now the final one.
 $CLI generate 1
-$LCLI1 dev-mocktime $(($EXPIRY + 33))
-$LCLI2 dev-mocktime $(($EXPIRY + 33))
+TIME=$(($EXPIRY + 33))
+$LCLI1 dev-mocktime $TIME
+$LCLI2 dev-mocktime $TIME
 sleep 1
 
 $LCLI1 getpeers | tr -s '\012\011 ' ' ' | fgrep '"peers" : [ ]'
@@ -167,7 +209,5 @@ $LCLI2 getpeers | tr -s '\012\011 ' ' ' | fgrep '"peers" : [ ]'
 
 $LCLI1 stop
 $LCLI2 stop
-scripts/shutdown.sh
-
-trap "rm -rf $DIR1 $DIR2" EXIT
 
+all_ok

daemon/watch.c

@@ -59,7 +59,8 @@ static void destroy_txowatch(struct txowatch *w)
 }
 
 /* Watch a txo. */
-static void insert_txo_watch(struct peer *peer,
+static void insert_txo_watch(const tal_t *ctx,
+			     struct peer *peer,
 			     const struct sha256_double *txid,
 			     unsigned int txout,
 			     void (*cb)(struct peer *peer,
@@ -67,7 +68,7 @@ static void insert_txo_watch(struct peer *peer,
 					void *cbdata),
 			     void *cbdata)
 {
-	struct txowatch *w = tal(peer, struct txowatch);
+	struct txowatch *w = tal(ctx, struct txowatch);
 
 	w->out.txid = *txid;
 	w->out.index = txout;
@@ -100,7 +101,6 @@ static void destroy_txwatch(struct txwatch *w)
 }
 
 static struct txwatch *insert_txwatch(const tal_t *ctx,
-				      struct lightningd_state *dstate,
 				      struct peer *peer,
 				      const struct sha256_double *txid,
 				      void (*cb)(struct peer *, int, void *),
@@ -109,7 +109,7 @@ static struct txwatch *insert_txwatch(const tal_t *ctx,
 	struct txwatch *w;
 
 	/* We could have a null-watch on it because we saw it spend a TXO */
-	w = txwatch_hash_get(&dstate->txwatches, txid);
+	w = txwatch_hash_get(&peer->dstate->txwatches, txid);
 	if (w) {
 		assert(!w->cb);
 		tal_free(w);
@@ -118,7 +118,7 @@ static struct txwatch *insert_txwatch(const tal_t *ctx,
 	w = tal(ctx, struct txwatch);
 	w->depth = 0;
 	w->txid = *txid;
-	w->dstate = dstate;
+	w->dstate = peer->dstate;
 	w->peer = peer;
 	w->cb = cb;
 	w->cbdata = cbdata;
@@ -129,7 +129,25 @@ static struct txwatch *insert_txwatch(const tal_t *ctx,
 	return w;
 }
 
-void add_anchor_watch_(struct peer *peer,
+/* This just serves to avoid us doing bitcoind_txid_lookup repeatedly
+ * on unknown txs. */
+static void insert_null_txwatch(struct lightningd_state *dstate,
+				const struct sha256_double *txid)
+{
+	struct txwatch *w = tal(dstate, struct txwatch);
+	w->depth = 0;
+	w->txid = *txid;
+	w->dstate = dstate;
+	w->peer = NULL;
+	w->cb = NULL;
+	w->cbdata = NULL;
+
+	txwatch_hash_add(&w->dstate->txwatches, w);
+	tal_add_destructor(w, destroy_txwatch);
+}
+
+void add_anchor_watch_(const tal_t *ctx,
+		       struct peer *peer,
 		       const struct sha256_double *txid,
 		       unsigned int out,
 		       void (*anchor_cb)(struct peer *peer, int depth, void *),
@@ -141,10 +159,10 @@ void add_anchor_watch_(struct peer *peer,
 	struct ripemd160 redeemhash;
 	u8 *redeemscript;
 
-	insert_txwatch(peer, peer->dstate, peer, txid, anchor_cb, cbdata);
-	insert_txo_watch(peer, txid, out, spend_cb, cbdata);
+	insert_txwatch(ctx, peer, txid, anchor_cb, cbdata);
+	insert_txo_watch(ctx, peer, txid, out, spend_cb, cbdata);
 
-	redeemscript = bitcoin_redeem_2of2(peer, &peer->them.commitkey,
+	redeemscript = bitcoin_redeem_2of2(ctx, &peer->them.commitkey,
 					   &peer->us.commitkey);
 	sha256(&h, redeemscript, tal_count(redeemscript));
 	ripemd160(&redeemhash, h.u.u8, sizeof(h));
@@ -156,12 +174,13 @@ void add_anchor_watch_(struct peer *peer,
 	bitcoind_watch_addr(peer->dstate, &redeemhash);
 }
 
-void add_commit_tx_watch_(struct peer *peer,
+void add_commit_tx_watch_(const tal_t *ctx,
+			  struct peer *peer,
 			  const struct sha256_double *txid,
 			  void (*cb)(struct peer *peer, int depth, void *),
 			  void *cbdata)
 {
-	insert_txwatch(peer, peer->dstate, peer, txid, cb, cbdata);
+	insert_txwatch(ctx, peer, txid, cb, cbdata);
 
 	/* We are already watching the anchor txo, so we don't need to
 	 * watch anything else. */
@@ -173,13 +192,14 @@ static void cb_no_arg(struct peer *peer, int depth, void *vcb)
 	cb(peer, depth);
 }
 
-void add_close_tx_watch(struct peer *peer,
+void add_close_tx_watch(const tal_t *ctx,
+			struct peer *peer,
 			const struct bitcoin_tx *tx,
 			void (*cb)(struct peer *peer, int depth))
 {
 	struct sha256_double txid;
 	bitcoin_txid(tx, &txid);
-	insert_txwatch(peer, peer->dstate, peer, &txid, cb_no_arg, cb);
+	insert_txwatch(ctx, peer, &txid, cb_no_arg, cb);
 
 	/* We are already watching the anchor txo, so we don't need to
 	 * watch anything else. */
@@ -223,7 +243,7 @@ static void watched_transaction(struct lightningd_state *dstate,
 	}
 
 	/* Don't report about this txid twice. */
-	insert_txwatch(dstate, dstate, NULL, txid, NULL, NULL);
+	insert_null_txwatch(dstate, txid);
 
 	/* Maybe it spent an output we're watching? */
 	if (!is_coinbase)

daemon/watch.h

@@ -61,7 +61,8 @@ HTABLE_DEFINE_TYPE(struct txwatch, txwatch_keyof, txid_hash, txwatch_eq,
 		   txwatch_hash);
 
 
-void add_anchor_watch_(struct peer *peer,
+void add_anchor_watch_(const tal_t *ctx,
+		       struct peer *peer,
 		       const struct sha256_double *txid,
 		       unsigned int out,
 		       void (*anchor_cb)(struct peer *peer, int depth, void *),
@@ -69,8 +70,8 @@ void add_anchor_watch_(struct peer *peer,
 					const struct bitcoin_tx *, void *),
 		       void *cbdata);
 
-#define add_anchor_watch(peer, txid, out, anchor_cb, spend_cb, cbdata)	\
-	add_anchor_watch_((peer), (txid), (out),			\
+#define add_anchor_watch(ctx, peer, txid, out, anchor_cb, spend_cb, cbdata) \
+	add_anchor_watch_((ctx), (peer), (txid), (out),			\
 			  typesafe_cb_preargs(void, void *,		\
 					      (anchor_cb), (cbdata),	\
 					      struct peer *,		\
@@ -81,20 +82,22 @@ void add_anchor_watch_(struct peer *peer,
 					      const struct bitcoin_tx *), \
 			  (cbdata))
 
-void add_commit_tx_watch_(struct peer *peer,
+void add_commit_tx_watch_(const tal_t *ctx,
+			  struct peer *peer,
 			  const struct sha256_double *txid,
 			  void (*cb)(struct peer *peer, int depth, void *),
 			  void *cbdata);
 
-#define add_commit_tx_watch(peer, txid, cb, cbdata) \
-	add_commit_tx_watch_((peer), (txid),			  \
+#define add_commit_tx_watch(ctx, peer, txid, cb, cbdata)		  \
+	add_commit_tx_watch_((ctx), (peer), (txid),			\
 			     typesafe_cb_preargs(void, void *,		\
 						 (cb), (cbdata),	\
 						 struct peer *,		\
 						 int depth),		\
 			     (cbdata))
 
-void add_close_tx_watch(struct peer *peer,
+void add_close_tx_watch(const tal_t *ctx,
+			struct peer *peer,
 			const struct bitcoin_tx *tx,
 			void (*cb)(struct peer *peer, int depth));