lightningd: move basic parameter parsing into common/configdir

lightning-cli is going to need to know what network we're on, so
it will need to parse the config files.  Move the code which does
the initial bootstrap parsing into common, as well as the config
file parsing core.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

cli/Makefile

@@ -2,6 +2,7 @@ LIGHTNING_CLI_SRC := cli/lightning-cli.c
 LIGHTNING_CLI_OBJS := $(LIGHTNING_CLI_SRC:.c=.o)
 
 LIGHTNING_CLI_COMMON_OBJS :=			\
+	bitcoin/chainparams.o			\
 	common/configdir.o			\
 	common/json.o				\
 	common/memleak.o			\

cli/lightning-cli.c

@@ -29,25 +29,6 @@
 #define ERROR_TALKING_TO_LIGHTNINGD 2
 #define ERROR_USAGE 3
 
-/* Tal wrappers for opt. */
-static void *opt_allocfn(size_t size)
-{
-	return tal_arr_label(NULL, char, size, TAL_LABEL("opt_allocfn", ""));
-}
-
-static void *tal_reallocfn(void *ptr, size_t size)
-{
-	if (!ptr)
-		return opt_allocfn(size);
-	tal_resize_(&ptr, 1, size, false);
-	return ptr;
-}
-
-static void tal_freefn(void *ptr)
-{
-	tal_free(ptr);
-}
-
 struct netaddr;
 
 /* Returns number of tokens digested */
@@ -450,8 +431,7 @@ int main(int argc, char *argv[])
 	jsmntok_t *toks;
 	const jsmntok_t *result, *error, *id;
 	const tal_t *ctx = tal(NULL, char);
-	char *lightning_dir = default_configdir(ctx);
-	char *rpc_filename = default_rpcfile(ctx);
+	char *config_filename, *lightning_dir, *rpc_filename;
 	jsmn_parser parser;
 	int parserr;
 	enum format format = DEFAULT_FORMAT;
@@ -462,15 +442,11 @@ int main(int argc, char *argv[])
 	jsmn_init(&parser);
 
 	tal_set_backend(NULL, NULL, NULL, tal_error);
-	opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn);
 
-	opt_register_arg("--lightning-dir=<dir>", opt_set_talstr, opt_show_charp,
-			 &lightning_dir,
-			 "Set working directory. All other files are relative to this");
+	setup_option_allocators();
 
-	opt_register_arg("--rpc-file", opt_set_talstr, opt_show_charp,
-			 &rpc_filename,
-			 "Set JSON-RPC socket (or /dev/tty)");
+	initial_config_opts(ctx, argc, argv,
+			    &config_filename, &lightning_dir, &rpc_filename);
 
 	opt_register_noarg("--help|-h", opt_usage_and_exit,
 			   "<command> [<params>...]", "Show this message. Use the command help (without hyphens -- \"lightning-cli help\") to get a list of all RPC commands");

cli/test/run-remove-hint.c

@@ -114,6 +114,7 @@ int main(int argc UNUSED, char *argv[])
 
 	output = tal_strdup(NULL, "");
 	assert(test_main(3, fake_argv) == 0);
+	assert(!taken_any());
 
 	assert(streq(output, "channels=\n"
 		     "\n"
@@ -129,5 +130,6 @@ int main(int argc UNUSED, char *argv[])
 		     "num_channels=1\n"
 		     "num_connected=1\n"));
 	tal_free(output);
+	take_cleanup();
 	return 0;
 }

common/configdir.c

@@ -1,17 +1,155 @@
 #include "configdir.h"
+#include <assert.h>
+#include <bitcoin/chainparams.h>
+#include <ccan/cast/cast.h>
+#include <ccan/err/err.h>
 #include <ccan/opt/opt.h>
+#include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
 #include <ccan/tal/str/str.h>
+#include <common/utils.h>
+#include <common/version.h>
 #include <errno.h>
 
+/* The regrettable globals */
+static const tal_t *options_ctx;
+
 /* Override a tal string; frees the old one. */
 char *opt_set_talstr(const char *arg, char **p)
 {
 	tal_free(*p);
-	return opt_set_charp(tal_strdup(NULL, arg), p);
+	return opt_set_charp(tal_strdup(options_ctx, arg), p);
+}
+
+static char *opt_set_abspath(const char *arg, char **p)
+{
+	tal_free(*p);
+	return opt_set_charp(path_join(options_ctx, take(path_cwd(NULL)), arg),
+			     p);
+}
+
+/* Tal wrappers for opt. */
+static void *opt_allocfn(size_t size)
+{
+	return tal_arr_label(NULL, char, size,
+			     TAL_LABEL(opt_allocfn_notleak, ""));
+}
+
+static void *tal_reallocfn(void *ptr, size_t size)
+{
+	if (!ptr)
+		return opt_allocfn(size);
+	tal_resize_(&ptr, 1, size, false);
+	return ptr;
+}
+
+static void tal_freefn(void *ptr)
+{
+	tal_free(ptr);
+}
+
+static int config_parse_line_number;
+
+static void config_log_stderr_exit(const char *fmt, ...)
+{
+	char *msg;
+	va_list ap;
+
+	va_start(ap, fmt);
+
+	/* This is the format we expect:*/
+	if (streq(fmt, "%s: %.*s: %s")) {
+		const char *argv0 = va_arg(ap, const char *);
+		unsigned int len = va_arg(ap, unsigned int);
+		const char *arg = va_arg(ap, const char *);
+		const char *problem = va_arg(ap, const char *);
+
+		assert(argv0 != NULL);
+		assert(arg != NULL);
+		assert(problem != NULL);
+		/*mangle it to remove '--' and add the line number.*/
+		msg = tal_fmt(NULL, "%s line %d: %.*s: %s",
+			      argv0,
+			      config_parse_line_number, len-2, arg+2, problem);
+	} else {
+		msg = tal_vfmt(NULL, fmt, ap);
+	}
+	va_end(ap);
+
+	errx(1, "%s", msg);
+}
+
+void parse_include(const char *filename, bool must_exist, bool early)
+{
+	char *contents, **lines;
+	char **all_args; /*For each line: either `--`argument, include file, or NULL*/
+	char *argv[3];
+	int i, argc;
+
+	contents = grab_file(NULL, filename);
+
+	/* The default config doesn't have to exist, but if the config was
+	 * specified on the command line it has to exist. */
+	if (!contents) {
+		if (must_exist)
+			err(1, "Opening and reading %s", filename);
+		return;
+	}
+
+	lines = tal_strsplit(contents, contents, "\r\n", STR_EMPTY_OK);
+
+	/* We have to keep all_args around, since opt will point into it: use
+	 * magic tal name to tell memleak this isn't one. */
+	all_args = tal_arr_label(options_ctx, char *, tal_count(lines) - 1,
+				 TAL_LABEL(options_array_notleak, ""));
+
+	for (i = 0; i < tal_count(lines) - 1; i++) {
+		if (strstarts(lines[i], "#")) {
+			all_args[i] = NULL;
+		} else if (strstarts(lines[i], "include ")) {
+			/* If relative, it's relative to current config file */
+			all_args[i] = path_join(all_args,
+						take(path_dirname(NULL,
+								  filename)),
+						lines[i] + strlen("include "));
+		} else {
+			/* Only valid forms are "foo" and "foo=bar" */
+			all_args[i] = tal_fmt(all_args, "--%s", lines[i]);
+		}
+	}
+
+	/*
+	For each line we construct a fake argc,argv commandline.
+	argv[1] is the only element that changes between iterations.
+	*/
+	argc = 2;
+	argv[0] = cast_const(char *, filename);
+	argv[argc] = NULL;
+
+	for (i = 0; i < tal_count(all_args); i++) {
+		if (all_args[i] == NULL)
+			continue;
+
+		if (!strstarts(all_args[i], "--")) {
+			parse_include(all_args[i], true, early);
+			continue;
+		}
+
+		config_parse_line_number = i + 1;
+		argv[1] = all_args[i];
+		if (early) {
+			opt_early_parse_incomplete(argc, argv,
+						   config_log_stderr_exit);
+		} else {
+			opt_parse(&argc, argv, config_log_stderr_exit);
+			argc = 2; /* opt_parse might have changed it  */
+		}
+	}
+
+	tal_free(contents);
 }
 
-char *default_configdir(const tal_t *ctx)
+static char *default_configdir(const tal_t *ctx)
 {
 	char *path;
 	const char *env = getenv("HOME");
@@ -22,7 +160,139 @@ char *default_configdir(const tal_t *ctx)
 	return path;
 }
 
-char *default_rpcfile(const tal_t *ctx)
+static char *default_rpcfile(const tal_t *ctx)
 {
 	return tal_strdup(ctx, "lightning-rpc");
 }
+
+static char *opt_set_network(const char *arg, void *unused)
+{
+	assert(arg != NULL);
+
+	/* Set the global chainparams instance */
+	chainparams = chainparams_for_network(arg);
+	if (!chainparams)
+		return tal_fmt(NULL, "Unknown network name '%s'", arg);
+	return NULL;
+}
+
+static char *opt_set_specific_network(const char *network)
+{
+	return opt_set_network(network, NULL);
+}
+
+static void opt_show_network(char buf[OPT_SHOW_LEN], const void *unused)
+{
+	snprintf(buf, OPT_SHOW_LEN, "%s", chainparams->network_name);
+}
+
+/* Special option to ignore stuff we've parsed really early on */
+char *opt_ignore(const char *arg, void *unused)
+{
+	return NULL;
+}
+
+char *opt_ignore_noarg(void *unused)
+{
+	return NULL;
+}
+
+void setup_option_allocators(void)
+{
+	/*~ These functions make ccan/opt use tal for allocations */
+	opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn);
+}
+
+void initial_config_opts(const tal_t *ctx,
+			 int argc, char *argv[],
+			 char **config_filename,
+			 char **config_dir,
+			 char **rpc_filename)
+{
+	options_ctx = ctx;
+
+	/* First, they could specify a config, which specifies a lightning dir
+	 * or a network. */
+	*config_filename = NULL;
+	opt_register_early_arg("--conf=<file>", opt_set_abspath, NULL,
+			       config_filename,
+			       "Specify configuration file (default: <lightning-dir>/config)");
+
+	/* Handle --version (and exit) here too */
+	opt_register_version();
+
+	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
+
+	/* Now, reset and ignore --conf option from now on. */
+	opt_free_table();
+
+	opt_register_early_arg("--conf=<file>", opt_ignore, NULL,
+			       config_filename,
+			       "Specify configuration file (default: <lightning-dir>/config)");
+
+	/* Now, config file (or cmdline) can set network and lightning-dir */
+	*config_dir = NULL;
+	opt_register_early_arg("--lightning-dir=<dir>",
+			       opt_set_talstr, NULL,
+			       config_dir,
+			       "Set working directory. All other files are relative to this");
+
+	/* We need to know network early, so we can set defaults (which normal
+	 * options can change) and default config_dir */
+	opt_register_early_arg("--network", opt_set_network, opt_show_network,
+			       NULL,
+			       "Select the network parameters (bitcoin, testnet,"
+			       " regtest, litecoin or litecoin-testnet)");
+	opt_register_early_noarg("--testnet",
+				 opt_set_specific_network, "testnet",
+				 "Alias for --network=testnet");
+	opt_register_early_noarg("--signet",
+				 opt_set_specific_network, "signet",
+				 "Alias for --network=signet");
+	opt_register_early_noarg("--mainnet",
+				 opt_set_specific_network, "bitcoin",
+				 "Alias for --network=bitcoin");
+
+	/* Read config file first, since cmdline must override */
+	if (*config_filename)
+		parse_include(*config_filename, true, true);
+	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
+
+	/* We use a global (in common/utils.h) for the chainparams.
+	 * We default to testnet for now. */
+	if (!chainparams)
+		chainparams = chainparams_for_network("testnet");
+
+	if (!*config_dir)
+		*config_dir = default_configdir(ctx);
+
+	/* Make sure it's absolute */
+	*config_dir = path_join(ctx, take(path_cwd(NULL)), take(*config_dir));
+
+	/* Now, reset and ignore those options from now on. */
+	opt_free_table();
+
+	opt_register_early_arg("--conf=<file>", opt_ignore, NULL,
+			       config_filename,
+			       "Specify configuration file (default: <lightning-dir>/config)");
+	opt_register_early_arg("--lightning-dir=<dir>",
+			       opt_ignore, opt_show_charp,
+			       config_dir,
+			       "Set working directory. All other files are relative to this");
+	opt_register_early_arg("--network", opt_ignore, opt_show_network,
+			       NULL,
+			       "Select the network parameters (bitcoin, testnet,"
+			       " regtest, litecoin or litecoin-testnet)");
+	opt_register_early_noarg("--testnet", opt_ignore_noarg, NULL,
+				 "Alias for --network=testnet");
+	opt_register_early_noarg("--signet", opt_ignore_noarg, NULL,
+				 "Alias for --network=signet");
+	opt_register_early_noarg("--mainnet", opt_ignore_noarg, NULL,
+				 "Alias for --network=bitcoin");
+
+	/* Set this up for when they parse cmdline proper. */
+	*rpc_filename = default_rpcfile(ctx);
+	opt_register_arg("--rpc-file", opt_set_talstr, opt_show_charp,
+			 rpc_filename,
+			 "Set JSON-RPC socket (or /dev/tty)");
+}

common/configdir.h

@@ -6,10 +6,21 @@
 /* Helper for options which are tal() strings. */
 char *opt_set_talstr(const char *arg, char **p);
 
-/* The default configuration dir: ~/.lightning */
-char *default_configdir(const tal_t *ctx);
+/* Initial options setup */
+void setup_option_allocators(void);
 
-/* The default rpc filename: lightning-rpc */
-char *default_rpcfile(const tal_t *ctx);
+/* Parse minimal config options and files */
+void initial_config_opts(const tal_t *ctx,
+			 int argc, char *argv[],
+			 char **config_filename,
+			 char **config_dir,
+			 char **rpc_filename);
+
+/* Parse a specific include file */
+void parse_include(const char *filename, bool must_exist, bool early);
+
+/* For listconfigs to access. */
+char *opt_ignore(const char *arg, void *unused);
+char *opt_ignore_noarg(void *unused);
 
 #endif /* LIGHTNING_COMMON_CONFIGDIR_H */

common/memleak.c

@@ -75,6 +75,10 @@ static void children_into_htable(const void *exclude1, const void *exclude2,
 			if (strends(name, "struct io_plan *[]") && !tal_parent(i))
 				continue;
 
+			/* Other notleak internals. */
+			if (strends(name, "_notleak"))
+				continue;
+
 			/* Don't add tmpctx. */
 			if (streq(name, "tmpctx"))
 				continue;

devtools/checkchannels.c

@@ -111,7 +111,7 @@ static void copy_column(void *dst, size_t size,
 
 int main(int argc, char *argv[])
 {
-	char *config_dir, *hsmfile, *dbfile;
+	char *config_dir, *config_filename, *rpc_filename, *hsmfile, *dbfile;
 	sqlite3 *sql;
 	sqlite3_stmt *stmt;
 	int flags = SQLITE_OPEN_READONLY, dberr;
@@ -123,11 +123,11 @@ int main(int argc, char *argv[])
 	wally_init(0);
 	secp256k1_ctx = wally_get_secp_context();
 
-	config_dir = default_configdir(NULL);
-	opt_register_arg("--lightning-dir=<dir>",
-			 opt_set_talstr, opt_show_charp,
-			 &config_dir,
-			 "Where to find hsm_secret and lightningd.sqlite3");
+	setup_option_allocators();
+
+	initial_config_opts(NULL, argc, argv,
+			    &config_filename, &config_dir, &rpc_filename);
+
 	opt_register_noarg("-v|--verbose", opt_set_bool, &verbose,
 			 "Print everything");
 

doc/lightning-cli.1

@@ -16,6 +16,17 @@ Set the directory for the lightning daemon we’re talking to; defaults to
 \fI$HOME/\.lightning\fR\.
 
 
+ \fB--conf\fR=\fIPATH\fR
+Sets configuration file (default: \fBlightning-dir\fR/\fIconfig\fR )\.
+
+
+ \fB--network\fR=\fInetwork\fR
+ \fB--mainnet\fR
+ \fB--testnet\fR
+ \fB--signet\fR
+Sets network explicitly\.
+
+
  \fB--rpc-file\fR=\fIFILE\fR
 Named pipe to use to talk to lightning daemon: default is
 \fIlightning-rpc\fR in the lightning directory\.

doc/lightning-cli.1.md

@@ -18,6 +18,15 @@ OPTIONS
 Set the directory for the lightning daemon we’re talking to; defaults to
 *$HOME/.lightning*.
 
+ **--conf**=*PATH*
+Sets configuration file (default: **lightning-dir**/*config* ).
+
+ **--network**=*network*
+ **--mainnet**
+ **--testnet**
+ **--signet**
+Sets network explicitly.
+
  **--rpc-file**=*FILE*
 Named pipe to use to talk to lightning daemon: default is
 *lightning-rpc* in the lightning directory.

lightningd/jsonrpc.c

@@ -1006,7 +1006,6 @@ void jsonrpc_setup(struct lightningd *ld)
 {
 	struct json_command **commands = get_cmdlist();
 
-	ld->rpc_filename = default_rpcfile(ld);
 	ld->jsonrpc = tal(ld, struct jsonrpc);
 	strmap_init(&ld->jsonrpc->usagemap);
 	ld->jsonrpc->commands = tal_arr(ld->jsonrpc, struct json_command *, 0);

lightningd/lightningd.c

@@ -205,8 +205,6 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	/*~ This is detailed in chaintopology.c */
 	ld->topology = new_topology(ld, ld->log);
 	ld->daemon_parent_fd = -1;
-	ld->config_filename = NULL;
-	ld->pidfile = NULL;
 	ld->proxyaddr = NULL;
 	ld->use_proxy_always = false;
 	ld->pure_tor_setup = false;
@@ -658,10 +656,6 @@ int main(int argc, char *argv[])
 	 * backtraces when we crash (if supported on this platform). */
 	daemon_setup(argv[0], log_backtrace_print, log_backtrace_exit);
 
-	/*~ We use a global (in common/utils.h) for the chainparams.
-	 * We default to testnet for now. */
-	chainparams = chainparams_for_network("testnet");
-
 	/*~ There's always a battle between what a constructor like this
 	 * should do, and what should be added later by the caller.  In
 	 * general, because we use valgrind heavily for testing, we prefer not

lightningd/options.c

@@ -7,7 +7,6 @@
 #include <ccan/read_write_all/read_write_all.h>
 #include <ccan/short_types/short_types.h>
 #include <ccan/str/hex/hex.h>
-#include <ccan/tal/grab_file/grab_file.h>
 #include <ccan/tal/path/path.h>
 #include <ccan/tal/str/str.h>
 #include <common/configdir.h>
@@ -44,38 +43,12 @@
 #include <wire/wire.h>
 
 bool deprecated_apis = true;
-static bool opt_table_alloced = false;
 
 /* Declare opt_add_addr here, because we we call opt_add_addr
  * and opt_announce_addr vice versa
 */
 static char *opt_add_addr(const char *arg, struct lightningd *ld);
 
-/* Tal wrappers for opt. */
-static void *opt_allocfn(size_t size)
-{
-	return tal_arr_label(NULL, char, size, TAL_LABEL("opt_allocfn", ""));
-}
-
-static void *tal_reallocfn(void *ptr, size_t size)
-{
-	if (!ptr) {
-		/* realloc(NULL) call is to allocate opt_table */
-		if (!opt_table_alloced) {
-			opt_table_alloced = true;
-			return notleak(opt_allocfn(size));
-		}
-		return opt_allocfn(size);
-	}
-	tal_resize_(&ptr, 1, size, false);
-	return ptr;
-}
-
-static void tal_freefn(void *ptr)
-{
-	tal_free(ptr);
-}
-
 /* FIXME: Put into ccan/time. */
 #define TIME_FROM_SEC(sec) { { .tv_nsec = 0, .tv_sec = sec } }
 #define TIME_FROM_MSEC(msec) \
@@ -250,38 +223,6 @@ static void opt_show_s32(char buf[OPT_SHOW_LEN], const s32 *u)
 	snprintf(buf, OPT_SHOW_LEN, "%"PRIi32, *u);
 }
 
-static char *opt_set_network(const char *arg, struct lightningd *ld)
-{
-	assert(arg != NULL);
-
-	/* Set the global chainparams instance */
-	chainparams = chainparams_for_network(arg);
-	if (!chainparams)
-		return tal_fmt(NULL, "Unknown network name '%s'", arg);
-	return NULL;
-}
-
-static char *opt_set_testnet(struct lightningd *ld)
-{
-	return opt_set_network("testnet", ld);
-}
-
-static char *opt_set_signet(struct lightningd *ld)
-{
-	return opt_set_network("signet", ld);
-}
-
-static char *opt_set_mainnet(struct lightningd *ld)
-{
-	return opt_set_network("bitcoin", ld);
-}
-
-static void opt_show_network(char buf[OPT_SHOW_LEN],
-			     const struct lightningd *ld)
-{
-	snprintf(buf, OPT_SHOW_LEN, "%s", chainparams->network_name);
-}
-
 static char *opt_set_rgb(const char *arg, struct lightningd *ld)
 {
 	assert(arg != NULL);
@@ -688,121 +629,6 @@ static void check_config(struct lightningd *ld)
 		fatal("--always-use-proxy needs --proxy");
 }
 
-static void setup_default_config(struct lightningd *ld)
-{
-	if (chainparams->testnet)
-		ld->config = testnet_config;
-	else
-		ld->config = mainnet_config;
-
-	/* Set default PID file name to be per-network */
-	tal_free(ld->pidfile);
-	ld->pidfile = tal_fmt(ld, "lightningd-%s.pid", chainparams->network_name);
-}
-
-static int config_parse_line_number;
-
-static void config_log_stderr_exit(const char *fmt, ...)
-{
-	char *msg;
-	va_list ap;
-
-	va_start(ap, fmt);
-
-	/* This is the format we expect:*/
-	if (streq(fmt, "%s: %.*s: %s")) {
-		const char *argv0 = va_arg(ap, const char *);
-		unsigned int len = va_arg(ap, unsigned int);
-		const char *arg = va_arg(ap, const char *);
-		const char *problem = va_arg(ap, const char *);
-
-		assert(argv0 != NULL);
-		assert(arg != NULL);
-		assert(problem != NULL);
-		/*mangle it to remove '--' and add the line number.*/
-		msg = tal_fmt(NULL, "%s line %d: %.*s: %s",
-			      argv0,
-			      config_parse_line_number, len-2, arg+2, problem);
-	} else {
-		msg = tal_vfmt(NULL, fmt, ap);
-	}
-	va_end(ap);
-
-	fatal("%s", msg);
-}
-
-static void parse_include(struct lightningd *ld,
-			  const char *filename,
-			  bool must_exist,
-			  bool early)
-{
-	char *contents, **lines;
-	char **all_args; /*For each line: either `--`argument, include file, or NULL*/
-	char *argv[3];
-	int i, argc;
-
-	contents = grab_file(ld, filename);
-
-	/* The default config doesn't have to exist, but if the config was
-	 * specified on the command line it has to exist. */
-	if (!contents) {
-		if (must_exist)
-			fatal("Opening and reading %s: %s",
-			      filename, strerror(errno));
-		return;
-	}
-
-	lines = tal_strsplit(contents, contents, "\r\n", STR_EMPTY_OK);
-
-	/* We have to keep all_args around, since opt will point into it */
-	all_args = notleak(tal_arr(ld, char *, tal_count(lines) - 1));
-
-	for (i = 0; i < tal_count(lines) - 1; i++) {
-		if (strstarts(lines[i], "#")) {
-			all_args[i] = NULL;
-		} else if (strstarts(lines[i], "include ")) {
-			/* If relative, it's relative to current config file */
-			all_args[i] = path_join(all_args,
-						take(path_dirname(NULL,
-								  filename)),
-						lines[i] + strlen("include "));
-		} else {
-			/* Only valid forms are "foo" and "foo=bar" */
-			all_args[i] = tal_fmt(all_args, "--%s", lines[i]);
-		}
-	}
-
-	/*
-	For each line we construct a fake argc,argv commandline.
-	argv[1] is the only element that changes between iterations.
-	*/
-	argc = 2;
-	argv[0] = cast_const(char *, filename);
-	argv[argc] = NULL;
-
-	for (i = 0; i < tal_count(all_args); i++) {
-		if (all_args[i] == NULL)
-			continue;
-
-		if (!strstarts(all_args[i], "--")) {
-			parse_include(ld, all_args[i], true, early);
-			continue;
-		}
-
-		config_parse_line_number = i + 1;
-		argv[1] = all_args[i];
-		if (early) {
-			opt_early_parse_incomplete(argc, argv,
-						   config_log_stderr_exit);
-		} else {
-			opt_parse(&argc, argv, config_log_stderr_exit);
-			argc = 2; /* opt_parse might have changed it  */
-		}
-	}
-
-	tal_free(contents);
-}
-
 /**
  * We turn the config file into cmdline arguments. @early tells us
  * whether to parse early options only (and ignore any unknown ones),
@@ -819,7 +645,7 @@ static void opt_parse_from_config(struct lightningd *ld, bool early)
 	else
 		filename = path_join(tmpctx, ld->config_dir, "config");
 
-	parse_include(ld, filename, ld->config_filename != NULL, early);
+	parse_include(filename, ld->config_filename != NULL, early);
 }
 
 static char *test_subdaemons_and_exit(struct lightningd *ld)
@@ -839,9 +665,6 @@ static char *list_features_and_exit(struct lightningd *ld)
 
 static char *opt_lightningd_usage(struct lightningd *ld)
 {
-	/* Reload config so that --help has the correct network defaults
-	 * to display before it exits */
-	setup_default_config(ld);
 	char *extra = tal_fmt(NULL, "\nA bitcoin lightning daemon (default "
 			"values shown for network: %s).", chainparams->network_name);
 	opt_usage_and_exit(extra);
@@ -884,73 +707,6 @@ static char *opt_start_daemon(struct lightningd *ld)
 	errx(1, "Died with signal %u", WTERMSIG(exitcode));
 }
 
-static char *opt_ignore_talstr(const char *arg, char **p)
-{
-	return NULL;
-}
-
-static char *opt_set_conf(const char *arg, struct lightningd *ld)
-{
-	/* This is a pass-through if arg is absolute */
-	tal_free(ld->config_filename);
-	ld->config_filename = path_join(ld, path_cwd(tmpctx), arg);
-	return NULL;
-}
-
-/* Just enough parsing to find config file, and other maintenance options
- * which don't want us to create the lightning dir */
-static void handle_minimal_config_opts(struct lightningd *ld,
-				       int argc, char *argv[])
-{
-	/* First, they could specify a config, which specifies a lightning dir */
-	opt_register_early_arg("--conf=<file>", opt_set_conf, NULL,
-			       ld,
-			       "Specify configuration file (default: <lightning-dir>/config)");
-
-	ld->config_dir = NULL;
-	opt_register_early_arg("--lightning-dir=<dir>",
-			       opt_set_talstr, NULL,
-			       &ld->config_dir,
-			       "Set working directory. All other files are relative to this");
-
-	/* List features immediately, before doing anything interesting */
-	opt_register_early_noarg("--list-features-only",
-				 list_features_and_exit,
-				 ld, opt_hidden);
-
-	/* Handle --version (and exit) here too: don't create lightning-dir for this */
-	opt_register_version();
-
-	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
-
-	/* Corner case: if they specified a config filename, and didn't set
-	 * set lightning-dir, read config file to get it! */
-	if (ld->config_filename && !ld->config_dir)
-		opt_parse_from_config(ld, true);
-
-	if (!ld->config_dir)
-		ld->config_dir = default_configdir(ld);
-
-	/* Now, reset and ignore those options from now on. */
-	opt_free_table();
-	opt_table_alloced = false;
-
-	opt_register_early_arg("--conf=<file>", opt_ignore_talstr, NULL,
-			       &ld->config_filename,
-			       "Specify configuration file (default: <lightning-dir>/config)");
-	opt_register_early_arg("--lightning-dir=<dir>",
-			       opt_ignore_talstr, opt_show_charp,
-			       &ld->config_dir,
-			       "Set working directory. All other files are relative to this");
-
-	ld->config_dir = path_join(ld, path_cwd(tmpctx), take(ld->config_dir));
-
-	ld->wallet_dsn = tal_fmt(ld, "sqlite3://%s/lightningd.sqlite3", ld->config_dir);
-	opt_register_early_arg("--wallet", opt_set_talstr, NULL,
-			       &ld->wallet_dsn,
-			       "Location of the wallet database.");
-}
-
 static void register_opts(struct lightningd *ld)
 {
 	/* This happens before plugins started */
@@ -971,19 +727,6 @@ static void register_opts(struct lightningd *ld)
 			       NULL, ld,
 			       "Disable a particular plugin by filename/name");
 
-	/* We need to know network early, so we can set defaults (which normal
-	 * options can change) */
-	opt_register_early_arg("--network", opt_set_network, opt_show_network,
-			       ld,
-			       "Select the network parameters (bitcoin, testnet,"
-			       " regtest, litecoin or litecoin-testnet)");
-	opt_register_early_noarg("--testnet", opt_set_testnet, ld,
-				 "Alias for --network=testnet");
-	opt_register_early_noarg("--signet", opt_set_signet, ld,
-				 "Alias for --network=signet");
-	opt_register_early_noarg("--mainnet", opt_set_mainnet, ld,
-				 "Alias for --network=bitcoin");
-
 	/* This can effect commandline parsing */
 	opt_register_early_arg("--allow-deprecated-apis",
 			       opt_set_bool_arg, opt_show_bool,
@@ -998,10 +741,10 @@ static void register_opts(struct lightningd *ld)
 	/* This immediately makes is a daemon. */
 	opt_register_early_noarg("--daemon", opt_start_daemon, ld,
 				 "Run in the background, suppress stdout/stderr");
+	opt_register_early_arg("--wallet", opt_set_talstr, NULL,
+			       &ld->wallet_dsn,
+			       "Location of the wallet database.");
 
-	opt_register_arg("--rpc-file", opt_set_talstr, opt_show_charp,
-			 &ld->rpc_filename,
-			 "Set JSON-RPC socket (or /dev/tty)");
 	opt_register_noarg("--help|-h", opt_lightningd_usage, ld,
 				 "Print this message.");
 	opt_register_arg("--bitcoin-datadir", opt_set_talstr, NULL,
@@ -1177,11 +920,33 @@ void setup_color_and_alias(struct lightningd *ld)
 
 void handle_early_opts(struct lightningd *ld, int argc, char *argv[])
 {
-	/*~ These functions make ccan/opt use tal for allocations */
-	opt_set_alloc(opt_allocfn, tal_reallocfn, tal_freefn);
+	/* Make ccan/opt use tal for allocations */
+	setup_option_allocators();
 
-	/*~ Handle --conf and --lightning-dir super-early. */
-	handle_minimal_config_opts(ld, argc, argv);
+	/*~ List features immediately, before doing anything interesting */
+	opt_register_early_noarg("--list-features-only",
+				 list_features_and_exit,
+				 ld, opt_hidden);
+
+	/*~ This does enough parsing to get us the base configuration options */
+	initial_config_opts(ld, argc, argv,
+			    &ld->config_filename,
+			    &ld->config_dir,
+			    &ld->rpc_filename);
+
+	/* Copy in default config, to be modified by further options */
+	if (chainparams->testnet)
+		ld->config = testnet_config;
+	else
+		ld->config = mainnet_config;
+
+	/* Now we can initialize wallet_dsn */
+	ld->wallet_dsn = tal_fmt(ld, "sqlite3://%s/lightningd.sqlite3",
+				 ld->config_dir);
+
+	/* Set default PID file name to be per-network */
+	ld->pidfile = tal_fmt(ld, "lightningd-%s.pid",
+			      chainparams->network_name);
 
 	/*~ Move into config dir: this eases path manipulation and also
 	 * gives plugins a good place to store their stuff. */
@@ -1207,9 +972,6 @@ void handle_early_opts(struct lightningd *ld, int argc, char *argv[])
 	/* Early cmdline options now override config file options. */
 	opt_early_parse_incomplete(argc, argv, opt_log_stderr_exit);
 
-	/* Now we know what network we're on, initialize defaults. */
-	setup_default_config(ld);
-
 	/* Finalize the logging subsystem now. */
 	logging_options_parsed(ld->log_book);
 }
@@ -1282,10 +1044,7 @@ static void add_config(struct lightningd *ld,
 			/* Ignore hidden options (deprecated) */
 		} else if (opt->cb == (void *)opt_usage_and_exit
 		    || opt->cb == (void *)version_and_exit
-		    /* These two show up as --network= */
-		    || opt->cb == (void *)opt_set_testnet
-		    || opt->cb == (void *)opt_set_signet
-		    || opt->cb == (void *)opt_set_mainnet
+		    || opt->cb == (void *)opt_ignore_noarg
 		    || opt->cb == (void *)opt_lightningd_usage
 		    || opt->cb == (void *)test_subdaemons_and_exit
 		    /* FIXME: we can't recover this. */
@@ -1335,7 +1094,7 @@ static void add_config(struct lightningd *ld,
 				answer = buf;
 		} else if (opt->cb_arg == (void *)opt_set_talstr
 			   || opt->cb_arg == (void *)opt_set_charp
-			   || opt->cb_arg == (void *)opt_ignore_talstr) {
+			   || opt->cb_arg == (void *)opt_ignore) {
 			const char *arg = *(char **)opt->u.carg;
 			if (arg)
 				answer = tal_fmt(name0, "%s", arg);

lightningd/test/run-jsonrpc.c

@@ -9,9 +9,6 @@ size_t bigsize_get(const u8 *p UNNEEDED, size_t max UNNEEDED, bigsize_t *val UNN
 /* Generated stub for bigsize_put */
 size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED)
 { fprintf(stderr, "bigsize_put called!\n"); abort(); }
-/* Generated stub for default_rpcfile */
-char *default_rpcfile(const tal_t *ctx UNNEEDED)
-{ fprintf(stderr, "default_rpcfile called!\n"); abort(); }
 /* Generated stub for fatal */
 void   fatal(const char *fmt UNNEEDED, ...)
 { fprintf(stderr, "fatal called!\n"); abort(); }