daemon: test restarting.

We add a "dev-restart" command which causes the daemon to close fds
and exec itself.  Then we do it after every command, with the caveat
that we always send a commit before newhtlc, because if not committed,
that is forgotten.  Fulfillhtlc and failhtlc get resent, since they're
idempotent.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -205,7 +205,13 @@ daemon-test-timeout-anchor\ --reconnect: daemon-test-different-fee-rates\ --reco
 daemon-test-different-fee-rates\ --reconnect: daemon-test-normal\ --reconnect
 daemon-test-normal\ --reconnect: daemon-test-manual-commit\ --reconnect
 daemon-test-manual-commit\ --reconnect: daemon-test-mutual-close-with-htlcs\ --reconnect
-daemon-test-mutual-close-with-htlcs\ --reconnect: daemon-all
+daemon-test-mutual-close-with-htlcs\ --reconnect: daemon-test-steal\ --restart
+daemon-test-steal\ --restart: daemon-test-dump-onchain\ --restart
+daemon-test-dump-onchain\ --restart: daemon-test-timeout-anchor\ --restart
+daemon-test-timeout-anchor\ --restart: daemon-test-different-fee-rates\ --restart
+daemon-test-different-fee-rates\ --restart: daemon-test-normal\ --restart
+daemon-test-normal\ --restart: daemon-test-mutual-close-with-htlcs\ --restart
+daemon-test-mutual-close-with-htlcs\ --restart: daemon-all
 daemon-tests: daemon-test-steal
 
 test-onion: test/test_onion test/onion_key

daemon/controlled_time.c

@@ -2,6 +2,8 @@
 #include "jsonrpc.h"
 #include "lightningd.h"
 #include "log.h"
+#include "opt_time.h"
+#include <ccan/tal/str/str.h>
 #include <inttypes.h>
 #include <stdio.h>
 
@@ -14,6 +16,22 @@ struct timeabs controlled_time(void)
 	return time_now();
 }
 
+void controlled_time_register_opts(void)
+{
+	opt_register_arg("--mocktime", opt_set_timeabs, opt_show_timeabs,
+			 &mock_time, opt_hidden);
+}
+
+char *controlled_time_arg(const tal_t *ctx)
+{
+	char buf[sizeof("--mocktime=") + OPT_SHOW_LEN] = "--mocktime=";
+	if (!mock_time.ts.tv_sec)
+		return NULL;
+
+	opt_show_timeabs(buf + strlen(buf), &mock_time);
+	return tal_strdup(ctx, buf);
+}
+
 static void json_mocktime(struct command *cmd,
 			  const char *buffer, const jsmntok_t *params)
 {

daemon/controlled_time.h

@@ -2,8 +2,11 @@
 #define LIGHTNING_DAEMON_CONTROLLED_TIME_H
 #include "config.h"
 #include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
 #include <ccan/time/time.h>
 
 struct timeabs controlled_time(void);
+void controlled_time_register_opts(void);
+char *controlled_time_arg(const tal_t *ctx);
 
 #endif /* LIGHTNING_DAEMON_CONTROLLED_TIME_H */

daemon/jsonrpc.c

@@ -1,9 +1,11 @@
 /* Code for JSON_RPC API */
 /* eg: { "method" : "dev-echo", "params" : [ "hello", "Arabella!" ], "id" : "1" } */
+#include "controlled_time.h"
 #include "json.h"
 #include "jsonrpc.h"
 #include "lightningd.h"
 #include "log.h"
+#include "peer.h"
 #include <ccan/array_size/array_size.h>
 #include <ccan/err/err.h>
 #include <ccan/io/io.h>
@@ -243,6 +245,37 @@ static const struct json_command crash_command = {
 	"Simple crash test for developers"
 };
 
+static void json_restart(struct command *cmd,
+			 const char *buffer, const jsmntok_t *params)
+{
+	const jsmntok_t *p, *end;
+	size_t n = 0;
+
+	if (params->type != JSMN_ARRAY) {
+		command_fail(cmd, "Need array to reexec");
+		return;
+	}
+	end = json_next(params);
+
+	cmd->dstate->reexec = tal_arrz(cmd->dstate, char *, n+1);
+	for (p = params + 1; p != end; p = json_next(p)) {
+		tal_resizez(&cmd->dstate->reexec, n+2);
+		cmd->dstate->reexec[n++] = tal_strndup(cmd->dstate->reexec,
+						       buffer + p->start,
+						       p->end - p->start);
+	}
+	debug_dump_peers(cmd->dstate);
+	io_break(cmd->dstate);
+	command_success(cmd, null_response(cmd));
+}	
+	
+static const struct json_command restart_command = {
+	"dev-restart",
+	json_restart,
+	"Re-exec the given {binary}.",
+	"Simple restart test for developers"
+};
+
 static const struct json_command *cmdlist[] = {
 	&help_command,
 	&stop_command,
@@ -262,6 +295,7 @@ static const struct json_command *cmdlist[] = {
 	&rhash_command,
 	&mocktime_command,
 	&crash_command,
+	&restart_command,
 	&disconnect_command,
 	&reconnect_command,
 	&signcommit_command,
@@ -340,6 +374,7 @@ void command_success(struct command *cmd, struct json_result *result)
 	}
 	assert(jcon->current == cmd);
 	json_result(jcon, cmd->id, json_result_string(result), "null");
+	log_debug(jcon->log, "Success");
 	jcon->current = tal_free(cmd);
 }
 
@@ -360,6 +395,8 @@ void command_fail(struct command *cmd, const char *fmt, ...)
 	error = tal_vfmt(cmd, fmt, ap);
 	va_end(ap);
 
+	log_debug(jcon->log, "Failing: %s", error);
+
 	/* Remove " */
 	while ((quote = strchr(error, '"')) != NULL)
 		*quote = '\'';

daemon/lightningd.c

@@ -141,9 +141,15 @@ static void config_register_opts(struct lightningd_state *dstate)
 	opt_register_arg("--fee-per-satoshi", opt_set_s32, opt_show_s32,
 			 &dstate->config.fee_per_satoshi,
 			 "Microsatoshi fee for every satoshi in HTLC");
-
 }
 
+static void dev_register_opts(struct lightningd_state *dstate)
+{
+	controlled_time_register_opts();
+	opt_register_noarg("--dev-no-routefail", opt_set_bool,
+			   &dstate->dev_never_routefail, opt_hidden);
+}
+	
 static void default_config(struct config *config)
 {
 	/* aka. "Dude, where's my coins?" */
@@ -248,6 +254,7 @@ static struct lightningd_state *lightningd_state(void)
 	dstate->dev_never_routefail = false;
 	dstate->bitcoin_req_running = false;
 	dstate->nodes = empty_node_map(dstate);
+	dstate->reexec = NULL;
 	return dstate;
 }
 
@@ -297,6 +304,7 @@ int main(int argc, char *argv[])
 	configdir_register_opts(dstate,
 				&dstate->config_dir, &dstate->rpc_filename);
 	config_register_opts(dstate);
+	dev_register_opts(dstate);
 
 	/* Get any configdir options first. */
 	opt_early_parse(argc, argv, opt_log_stderr_exit);
@@ -365,6 +373,35 @@ int main(int argc, char *argv[])
 			cleanup_peers(dstate);
 	}
 
+	if (dstate->reexec) {
+		int fd;
+		char *mocktimearg;
+
+		log_unusual(dstate->base_log, "Restart at user request");
+		fflush(stdout);
+		fflush(stderr);
+
+		/* Manually close all fds (or near enough!) */
+		for (fd = 3; fd < 1024; fd++)
+			close(fd);
+
+		/* Maybe append mocktime arg. */
+		mocktimearg = controlled_time_arg(dstate->reexec);
+		if (mocktimearg) {
+			size_t n = tal_count(dstate->reexec);
+			tal_resizez(&dstate->reexec, n+1);
+			dstate->reexec[n-1] = mocktimearg;
+		}
+		if (dstate->dev_never_routefail) {
+			size_t n = tal_count(dstate->reexec);
+			tal_resizez(&dstate->reexec, n+1);
+			dstate->reexec[n-1] = "--dev-no-routefail";
+		}
+		execvp(dstate->reexec[0], dstate->reexec);
+		fatal("Exec '%s' failed: %s",
+		      dstate->reexec[0], strerror(errno));
+	}
+
 	tal_free(dstate);
 	opt_free_table();
 	return 0;

daemon/lightningd.h

@@ -112,5 +112,8 @@ struct lightningd_state {
 
 	/* For testing: don't fail if we can't route. */
 	bool dev_never_routefail;
+
+	/* Re-exec hack for testing. */
+	char **reexec;
 };
 #endif /* LIGHTNING_DAEMON_LIGHTNING_H */

daemon/opt_time.c

@@ -77,3 +77,22 @@ void opt_show_time(char buf[OPT_SHOW_LEN], const struct timerel *t)
 	} else
 		sprintf(buf, "%lus", t->ts.tv_sec);
 }
+
+char *opt_set_timeabs(const char *arg, struct timeabs *t)
+{
+	long double d;
+
+	if (sscanf(arg, "%Lf", &d) != 1)
+		return tal_fmt(NULL, "'%s' is not a time", arg);
+	t->ts.tv_sec = d;
+	t->ts.tv_nsec = (d - t->ts.tv_sec) * 1000000000;
+	return NULL;
+}
+
+void opt_show_timeabs(char buf[OPT_SHOW_LEN], const struct timeabs *t)
+{
+	long double d = t->ts.tv_sec;
+	d = d * 1000000000 + t->ts.tv_nsec;
+
+	sprintf(buf, "%.9Lf", d);
+}

daemon/opt_time.h

@@ -7,4 +7,7 @@
 char *opt_set_time(const char *arg, struct timerel *t);
 void opt_show_time(char buf[OPT_SHOW_LEN], const struct timerel *t);
 
+char *opt_set_timeabs(const char *arg, struct timeabs *t);
+void opt_show_timeabs(char buf[OPT_SHOW_LEN], const struct timeabs *t);
+
 #endif /* LIGHTNING_DAEMON_OPT_TIME_H */

daemon/test/test.sh

@@ -72,7 +72,10 @@ while [ $# != 0 ]; do
 	x"--normal")
 	    ;;
 	x"--reconnect")
-	    RECONNECT=1
+	    RECONNECT=reconnect
+	    ;;
+	x"--restart")
+	    RECONNECT=restart
 	    ;;
 	x"--crash")
 	    CRASH_ON_FAIL=1
@@ -93,9 +96,11 @@ LCLI3="../lightning-cli --lightning-dir=$DIR3"
 
 if [ -n "$VERBOSE" ]; then
     FGREP="fgrep"
+    SHOW="cat >&2"
 else
     # Suppress command output.
     exec >/dev/null
+    SHOW="cat"
 fi
 
 lcli1()
@@ -112,12 +117,45 @@ lcli1()
 	    ;;
 	    dev-mocktime*)
 	    ;;
+	    dev-disconnect)
+	    ;;
 	    stop)
 	    ;;
 	    *)
-		[ -z "$VERBOSE" ] || echo RECONNECTING >&2
+		case "$RECONNECT" in
-		$LCLI1 dev-reconnect $ID2 >/dev/null 
+		    reconnect)
-		sleep 1
+			[ -z "$VERBOSE" ] || echo RECONNECTING >&2
+			$LCLI1 dev-reconnect $ID2 >/dev/null
+			sleep 1
+			;;
+		    restart)
+			[ -z "$VERBOSE" ] || echo RESTARTING >&2
+			# FIXME: Instead, check if command was committed, and
+			# if not, resubmit!
+			if [ "$1" = "newhtlc" ]; then
+			    $LCLI1 commit $ID2 >/dev/null 2>&1 || true
+			fi
+			$LCLI1 -- dev-restart $LIGHTNINGD1 >/dev/null 2>&1 || true
+			if ! check "$LCLI1 getlog 2>/dev/null | fgrep -q Hello"; then
+			    echo "dev-restart failed!">&2
+			    exit 1
+			fi
+			# It will have forgotten any added routes.
+			if [ -n "$ADDROUTE" ]; then
+			    echo $LCLI1 $ADDROUTE >&2
+			    $LCLI1 $ADDROUTE >&2
+			fi
+			# These are safe to resubmit, will simply fail.
+			if [ "$1" = "fulfillhtlc" -o "$1" = "failhtlc" ]; then
+			    if [ -z "$VERBOSE" ]; then
+				$LCLI1 "$@" >/dev/null 2>&1 || true
+			    else
+				echo "Rerunning $LCLI1 $@" >&2
+				$LCLI1 "$@" 2>&1 || true
+			    fi
+			fi
+			;;
+		esac
 		;;
 	esac
     fi
@@ -324,11 +362,15 @@ if [ -n "$DIFFERENT_FEES" ]; then
     echo "default-fee-rate=$CLOSE_FEE_RATE2" >> $DIR2/config
 fi
 
+# Need absolute path for re-exec testing.
+LIGHTNINGD1="$(readlink -f `pwd`/../lightningd) --lightning-dir=$DIR1"
 if [ -n "$GDB1" ]; then
-    echo Press return once you run: gdb --args daemon/lightningd --lightning-dir=$DIR1 >&2
+    echo Press return once you run: gdb --args $LIGHTNINGD1 >&2
+    
     read REPLY
 else
-    $PREFIX ../lightningd --lightning-dir=$DIR1 > $REDIR1 2> $REDIRERR1 &
+    LIGHTNINGD1="$PREFIX $LIGHTNINGD1"
+    $LIGHTNINGD1 > $REDIR1 2> $REDIRERR1 &
 fi
 
 if [ -n "$GDB2" ]; then
@@ -869,7 +911,8 @@ if [ ! -n "$MANUALCOMMIT" ]; then
     HTLC_AMOUNT=100000000
 
     # Tell node 1 about the 2->3 route.
-    lcli1 add-route $ID2 $ID3 546000 10 36 36
+    ADDROUTE="add-route $ID2 $ID3 546000 10 36 36"
+    lcli1 $ADDROUTE
     RHASH5=`lcli3 accept-payment $HTLC_AMOUNT | sed 's/.*"\([0-9a-f]*\)".*/\1/'`
 
     # Try wrong hash.