Force the use of the POSIX C locale for all commands and their subprocesses

Makefile

@@ -230,7 +230,7 @@ bolt-precheck:
 
 check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c)
 
-tools/check-bolt: tools/check-bolt.o $(CCAN_OBJS)
+tools/check-bolt: tools/check-bolt.o $(CCAN_OBJS) common/utils.o
 
 tools/check-bolt.o: $(CCAN_HEADERS)
 
@@ -265,7 +265,10 @@ check-cppcheck: .cppcheck-suppress
 check-shellcheck:
 	git ls-files -- "*.sh" | xargs shellcheck
 
-check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck
+check-setup_locale:
+	@tools/check-setup_locale.sh
+
+check-source: check-makefile check-source-bolt check-whitespace check-markdown check-spelling check-python check-includes check-cppcheck check-shellcheck check-setup_locale
 
 full-check: check check-source
 

bitcoin/test/run-tx-encode.c

@@ -26,6 +26,8 @@ static void tal_hexeq(const u8 *p, const char *hex)
 
 int main(void)
 {
+	setup_locale();
+
 	struct bitcoin_tx *tx;
 
 	tx = bitcoin_tx_from_hex(NULL, extended_tx, strlen(extended_tx));

channeld/channel.c

@@ -2544,6 +2544,8 @@ static void send_shutdown_complete(struct peer *peer)
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	int i, nfds;
 	fd_set fds_in, fds_out;
 	struct peer *peer;

channeld/test/run-full_channel.c

@@ -317,6 +317,8 @@ static void update_feerate(struct channel *channel, u32 feerate)
 
 int main(void)
 {
+	setup_locale();
+
 	struct bitcoin_txid funding_txid;
 	/* We test from both sides. */
 	struct channel *lchannel, *rchannel;

cli/Makefile

@@ -5,6 +5,7 @@ LIGHTNING_CLI_COMMON_OBJS :=			\
 	common/configdir.o			\
 	common/json.o				\
 	common/json_escaped.o			\
+	common/utils.o			\
 	common/version.o
 
 lightning-cli-all: cli/lightning-cli

cli/lightning-cli.c

@@ -8,6 +8,7 @@
 #include <ccan/tal/str/str.h>
 #include <common/configdir.h>
 #include <common/json.h>
+#include <common/utils.h>
 #include <common/version.h>
 #include <stdio.h>
 #include <sys/socket.h>
@@ -167,6 +168,8 @@ static void add_input(char **cmd, const char *input,
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	int fd, i, off;
 	const char *method;
 	char *cmd, *resp, *idstr, *rpc_filename;

cli/test/run-large-input.c

@@ -83,8 +83,9 @@ ssize_t test_read(int fd UNUSED, void *buf, size_t len)
 
 int main(int argc UNUSED, char *argv[])
 {
-	char *fake_argv[] = { argv[0], "--lightning-dir=/tmp/", "test", NULL };
+	setup_locale();
 
+	char *fake_argv[] = { argv[0], "--lightning-dir=/tmp/", "test", NULL };
 
 	/* sizeof() is an overestimate, but we don't care. */
 	response = tal_arr(NULL, char,

closingd/closing.c

@@ -423,6 +423,8 @@ static u64 adjust_offer(struct crypto_state *cs,
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct crypto_state cs;
 	const tal_t *ctx = tal(NULL, char);
 	u8 *msg;

common/test/run-bolt11.c

@@ -115,6 +115,8 @@ static void test_b11(const char *b11str,
 
 int main(void)
 {
+	setup_locale();
+
 	struct bolt11 *b11;
 	struct pubkey node;
 	u64 msatoshi;

common/test/run-ip_port_parsing.c

@@ -27,6 +27,8 @@ void towire_u8(u8 **pptr UNNEEDED, u8 v UNNEEDED)
 
 int main(void)
 {
+	setup_locale();
+
 	struct wireaddr addr;
 	char *ip;
 	u16 port;

common/test/run-json.c

@@ -1,5 +1,6 @@
 #include "../json.c"
 #include "../json_escaped.c"
+#include <common/utils.h>
 #include <stdio.h>
 
 /* AUTOGENERATED MOCKS START */
@@ -146,6 +147,8 @@ static void test_json_partial(void)
 
 int main(void)
 {
+	setup_locale();
+
 	test_json_tok_bitcoin_amount();
 	test_json_filter();
 	test_json_escape();

common/test/run-sphinx.c

@@ -158,6 +158,8 @@ static void run_unit_tests(void)
 
 int main(int argc, char **argv)
 {
+	setup_locale();
+
 	bool generate = false, decode = false, unit = false;
 	u8 assocdata[32];
 	memset(assocdata, 'B', sizeof(assocdata));

common/utils.c

@@ -2,6 +2,7 @@
 #include <ccan/list/list.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
+#include <locale.h>
 
 secp256k1_context *secp256k1_ctx;
 const tal_t *tmpctx;
@@ -26,6 +27,13 @@ u8 *tal_hexdata(const tal_t *ctx, const void *str, size_t len)
 	return data;
 }
 
+/* Use the POSIX C locale. */
+void setup_locale(void)
+{
+	setlocale(LC_ALL, "C");
+	putenv("LC_ALL=C"); /* For exec{l,lp,v,vp}(...) */
+}
+
 /* Global temporary convenience context: freed in io loop core. */
 
 /* Initial creation of tmpctx. */

common/utils.h

@@ -16,6 +16,9 @@ char *tal_hex(const tal_t *ctx, const tal_t *data);
 /* Allocate and fill a buffer with the data of this hex string. */
 u8 *tal_hexdata(const tal_t *ctx, const void *str, size_t len);
 
+/* Use the POSIX C locale. */
+void setup_locale(void);
+
 /* Global temporary convenience context: freed in io loop core. */
 extern const tal_t *tmpctx;
 

devtools/bolt11-cli.c

@@ -54,6 +54,8 @@ static char *fmt_time(const tal_t *ctx, u64 time)
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	const tal_t *ctx = tal(NULL, char);
 	const char *method;
 	struct bolt11 *b11;

devtools/decodemsg.c

@@ -3,6 +3,8 @@
 
 int main(int argc UNUSED, char *argv[])
 {
+	setup_locale();
+
 	u8 *m = tal_hexdata(NULL, argv[1], strlen(argv[1]));
 	print_message(m);
 	return 0;

devtools/onion.c

@@ -106,6 +106,8 @@ static void do_decode(int argc, char **argv)
 
 int main(int argc, char **argv)
 {
+	setup_locale();
+
 	bool generate = false, decode = false;
 	secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY |
 						 SECP256K1_CONTEXT_SIGN);

gossipd/gossip.c

@@ -2362,6 +2362,8 @@ static void master_gone(struct io_conn *unused UNUSED, struct daemon_conn *dc UN
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct daemon *daemon;
 
 	subdaemon_setup(argc, argv);

gossipd/test/run-bench-find_route.c

@@ -203,6 +203,8 @@ static void run(const char *name)
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	static const struct bitcoin_blkid zerohash;
 	struct routing_state *rstate;
 	size_t num_nodes = 100, num_runs = 1;

gossipd/test/run-find_route-specific.c

@@ -128,6 +128,8 @@ static bool channel_is_between(const struct chan *chan,
 
 int main(void)
 {
+	setup_locale();
+
 	static const struct bitcoin_blkid zerohash;
 	struct half_chan *nc;
 	struct routing_state *rstate;

gossipd/test/run-find_route.c

@@ -172,6 +172,8 @@ static bool channel_is_between(const struct chan *chan,
 
 int main(void)
 {
+	setup_locale();
+
 	static const struct bitcoin_blkid zerohash;
 	struct routing_state *rstate;
 	struct pubkey a, b, c, d;

gossipd/test/run-initiator-success.c

@@ -199,6 +199,8 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point)
 
 int main(void)
 {
+	setup_locale();
+
 	struct wireaddr dummy;
 
 	secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY

gossipd/test/run-responder-success.c

@@ -196,6 +196,8 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point)
 
 int main(void)
 {
+	setup_locale();
+
 	struct wireaddr dummy;
 
 	secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY

hsmd/hsm.c

@@ -824,6 +824,8 @@ static void master_gone(struct io_conn *unused UNUSED, struct daemon_conn *dc UN
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct client *client;
 
 	subdaemon_setup(argc, argv);

lightningd/lightningd.c

@@ -280,6 +280,8 @@ static void pidfile_create(const struct lightningd *ld)
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct lightningd *ld;
 	bool newdir;
 	u32 blockheight;

lightningd/test/run-commit_tx.c

@@ -425,6 +425,8 @@ static const struct htlc **invert_htlcs(const struct htlc **htlcs)
 
 int main(void)
 {
+	setup_locale();
+
 	struct bitcoin_txid funding_txid;
 	u64 funding_amount_satoshi, dust_limit_satoshi;
 	u32 feerate_per_kw;

lightningd/test/run-cryptomsg.c

@@ -98,6 +98,8 @@ static struct secret secret_from_hex(const char *hex)
 
 int main(void)
 {
+	setup_locale();
+
 	struct peer_crypto_state cs_out, cs_in;
 	struct secret sk, rk, ck;
 	const void *msg;

lightningd/test/run-find_my_path.c

@@ -157,6 +157,8 @@ struct log *crashlog;
 #undef main
 int main(int argc UNUSED, char *argv[] UNUSED)
 {
+	setup_locale();
+
 	char *argv0;
 	/* We're assuming we're run from top build dir. */
 	const char *answer;

lightningd/test/run-funding_tx.c

@@ -41,6 +41,8 @@ static struct privkey privkey_from_hex(const char *hex)
 
 int main(void)
 {
+	setup_locale();
+
 	struct bitcoin_tx *input, *funding;
 	u64 fee;
 	struct pubkey local_funding_pubkey, remote_funding_pubkey;

lightningd/test/run-key_derive.c

@@ -18,6 +18,8 @@ static struct secret secret_from_hex(const char *hex)
 
 int main(void)
 {
+	setup_locale();
+
 	struct privkey privkey;
 	struct secret base_secret, per_commitment_secret;
 	struct pubkey base_point, per_commitment_point, pubkey, pubkey2;

onchaind/onchain.c

@@ -2096,6 +2096,8 @@ static void handle_their_unilateral(const struct bitcoin_tx *tx,
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	const tal_t *ctx = tal(NULL, char);
 	u8 *msg;
 	struct privkey seed;

onchaind/test/run-grind_feerate.c

@@ -153,6 +153,8 @@ bool wire_sync_write(int fd UNNEEDED, const void *msg TAKES UNNEEDED)
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct bitcoin_tx *tx;
 	secp256k1_ecdsa_signature sig;
 	u8 *der, *wscript;

openingd/opening.c

@@ -793,6 +793,8 @@ static u8 *fundee_channel(struct state *state,
 #ifndef TESTING
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	u8 *msg, *peer_msg;
 	struct state *state = tal(NULL, struct state);
 	struct privkey seed;

tools/check-bolt.c

@@ -7,6 +7,7 @@
 #include <ccan/tal/path/path.h>
 #include <ccan/tal/str/str.h>
 #include <ccan/tal/tal.h>
+#include <common/utils.h>
 #include <sys/types.h>
 #include <dirent.h>
 
@@ -256,6 +257,8 @@ static struct bolt_file *find_bolt(const char *bolt_prefix,
 
 int main(int argc, char *argv[])
 {
+	setup_locale();
+
 	struct bolt_file *bolts;
 	int i;
 

tools/check-setup_locale.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+EXIT_CODE=0
+for FILE in $(git grep -lE 'int main\(' | grep -vE '^ccan/'); do
+    if ! grep -q 'setup_locale();' "${FILE}"; then
+        echo "main(...) in ${FILE} does not call setup_locale() (see common/utils.h)"
+        EXIT_CODE=1
+    fi
+done
+if [[ ${EXIT_CODE} != 0 ]]; then
+    echo
+    echo "setup_locale() forces the use of the POSIX C locale. By using the"
+    echo "POSIX C locale we avoid a class of localization related parsing bugs"
+    echo "that can be very tricky to isolate and fix."
+fi
+exit ${EXIT_CODE}

wallet/test/run-db.c

@@ -116,6 +116,8 @@ static bool test_vars(void)
 
 int main(void)
 {
+	setup_locale();
+
 	bool ok = true;
 
 	ok &= test_empty_db_migrate();

wallet/test/run-wallet.c

@@ -992,6 +992,8 @@ static bool test_payment_crud(struct lightningd *ld, const tal_t *ctx)
 
 int main(void)
 {
+	setup_locale();
+
 	bool ok = true;
 	struct lightningd *ld;
 

wire/test/run-peer-wire.c

@@ -844,6 +844,8 @@ static bool node_announcement_eq(const struct msg_node_announcement *a,
 
 int main(void)
 {
+	setup_locale();
+
 	struct msg_channel_announcement ca, *ca2;
 	struct msg_funding_locked fl, *fl2;
 	struct msg_announcement_signatures as, *as2;