bitcoind: handle up to 60 seconds of outage.

Seems to go out to lunch on reorgs:

   +136792.168286138 lightningd(9465):BROKEN: bitcoin-cli getchaintips exited 28: 'error code: -28
   error message:
   Rewinding blocks...

Closes: #286
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

lightningd/bitcoind.c

@@ -117,14 +117,34 @@ static void bcli_finished(struct io_conn *conn, struct bitcoin_cli *bcli)
 
 	if (!bcli->exitstatus) {
 		if (WEXITSTATUS(status) != 0) {
-			fatal("%s exited %u: '%.*s'", bcli_args(bcli),
-			      WEXITSTATUS(status),
-			      (int)bcli->output_bytes,
-			      bcli->output);
+			/* Allow 60 seconds of spurious errors, eg. reorg. */
+			struct timerel t;
+
+			log_unusual(bcli->bitcoind->log,
+				    "%s exited with status %u",
+				    bcli_args(bcli),
+				    WEXITSTATUS(status));
+
+			if (!bitcoind->error_count)
+				bitcoind->first_error_time = time_mono();
+
+			t = timemono_between(time_mono(),
+					     bitcoind->first_error_time);
+			if (time_greater(t, time_from_sec(60)))
+				fatal("%s exited %u (after %u other errors) '%.*s'",
+				      bcli_args(bcli),
+				      WEXITSTATUS(status),
+				      bitcoind->error_count,
+				      (int)bcli->output_bytes,
+				      bcli->output);
+			bitcoind->error_count++;
 		}
 	} else
 		*bcli->exitstatus = WEXITSTATUS(status);
 
+	if (WEXITSTATUS(status) == 0)
+		bitcoind->error_count = 0;
+
 	bitcoind->req_running = false;
 
 	/* Don't continue if were only here because we were freed for shutdown */
@@ -154,7 +174,6 @@ static void next_bcli(struct bitcoind *bitcoind)
 
 	bitcoind->req_running = true;
 	conn = io_new_conn(bitcoind, bcli->fd, output_init, bcli);
-	tal_steal(conn, bcli);
 	io_set_finish(conn, bcli_finished, bcli);
 }
 
@@ -526,6 +545,7 @@ struct bitcoind *new_bitcoind(const tal_t *ctx, struct log *log)
 	bitcoind->log = log;
 	bitcoind->req_running = false;
 	bitcoind->shutdown = false;
+	bitcoind->error_count = 0;
 	list_head_init(&bitcoind->pending);
 	tal_add_destructor(bitcoind, destroy_bitcoind);
 

lightningd/bitcoind.h

@@ -5,6 +5,7 @@
 #include <ccan/list/list.h>
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
+#include <ccan/time/time.h>
 #include <ccan/typesafe_cb/typesafe_cb.h>
 #include <stdbool.h>
 
@@ -37,6 +38,10 @@ struct bitcoind {
 	/* What network are we on? */
 	const struct chainparams *chainparams;
 
+	/* If non-zero, time we first hit a bitcoind error. */
+	unsigned int error_count;
+	struct timemono first_error_time;
+
 	/* Ignore results, we're shutting down. */
 	bool shutdown;
 };