common/json: move JSON creation routines into lightningd/

It's the only user of them, and it's going to get optimized.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

gossip.pydiff --git a/common/test/run-json.c b/common/test/run-json.c
index 956fdda35..db52d6b01 100644

cli/Makefile

@@ -4,8 +4,7 @@ LIGHTNING_CLI_OBJS := $(LIGHTNING_CLI_SRC:.c=.o)
 LIGHTNING_CLI_COMMON_OBJS :=			\
 	common/configdir.o			\
 	common/json.o				\
-	common/json_escaped.o			\
+	common/utils.o				\
-	common/utils.o			\
 	common/version.o
 
 lightning-cli-all: cli/lightning-cli

cli/test/Makefile

@@ -12,7 +12,6 @@ CLI_TEST_COMMON_OBJS :=				\
 	common/daemon_conn.o			\
 	common/htlc_state.o			\
 	common/json.o				\
-	common/json_escaped.o			\
 	common/pseudorand.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\

common/Makefile

@@ -25,7 +25,6 @@ COMMON_SRC_NOGEN :=				\
 	common/initial_commit_tx.c		\
 	common/io_lock.c			\
 	common/json.c				\
-	common/json_escaped.c			\
 	common/key_derive.c			\
 	common/keyset.c				\
 	common/memleak.c			\

common/json.c

@@ -1,6 +1,5 @@
 /* JSON core and helpers */
 #include "json.h"
-#include "json_escaped.h"
 #include <assert.h>
 #include <ccan/build_assert/build_assert.h>
 #include <ccan/str/hex/hex.h>
@@ -11,14 +10,6 @@
 #include <stdio.h>
 #include <string.h>
 
-struct json_result {
-	/* tal_arr of types we're enclosed in. */
-	jsmntype_t *wrapping;
-
-	/* tal_count() of this is strlen() + 1 */
-	char *s;
-};
-
 const char *json_tok_contents(const char *buffer, const jsmntok_t *t)
 {
 	if (t->type == JSMN_STRING)
@@ -225,231 +216,3 @@ again:
 
 	return toks;
 }
-
-static void result_append(struct json_result *res, const char *str)
-{
-	size_t len = tal_count(res->s) - 1;
-
-	tal_resize(&res->s, len + strlen(str) + 1);
-	strcpy(res->s + len, str);
-}
-
-static void PRINTF_FMT(2,3)
-result_append_fmt(struct json_result *res, const char *fmt, ...)
-{
-	size_t len = tal_count(res->s) - 1, fmtlen;
-	va_list ap;
-
-	va_start(ap, fmt);
-	fmtlen = vsnprintf(NULL, 0, fmt, ap);
-	va_end(ap);
-
-	tal_resize(&res->s, len + fmtlen + 1);
-	va_start(ap, fmt);
-	vsprintf(res->s + len, fmt, ap);
-	va_end(ap);
-}
-
-static bool result_ends_with(struct json_result *res, const char *str)
-{
-	size_t len = tal_count(res->s) - 1;
-
-	if (strlen(str) > len)
-		return false;
-	return streq(res->s + len - strlen(str), str);
-}
-
-static void check_fieldname(const struct json_result *result,
-			    const char *fieldname)
-{
-	size_t n = tal_count(result->wrapping);
-	if (n == 0)
-		/* Can't have a fieldname if not in anything! */
-		assert(!fieldname);
-	else if (result->wrapping[n-1] == JSMN_ARRAY)
-		/* No fieldnames in arrays. */
-		assert(!fieldname);
-	else
-		/* Must have fieldnames in objects. */
-		assert(fieldname);
-}
-
-static void result_add_indent(struct json_result *result);
-
-	static void json_start_member(struct json_result *result, const char *fieldname)
-{
-	/* Prepend comma if required. */
-	if (result->s[0]
-	    && !result_ends_with(result, "{")
-	    && !result_ends_with(result, "["))
-		result_append(result, ", \n");
-	else
-		result_append(result, "\n");
-
-	result_add_indent(result);
-
-	check_fieldname(result, fieldname);
-	if (fieldname)
-		result_append_fmt(result, "\"%s\": ", fieldname);
-}
-
-static void result_add_indent(struct json_result *result)
-{
-	size_t i, indent = tal_count(result->wrapping);
-
-	if (!indent)
-		return;
-
-	for (i = 0; i < indent; i++)
-		result_append(result, "  ");
-}
-
-static void result_add_wrap(struct json_result *result, jsmntype_t type)
-{
-	size_t indent = tal_count(result->wrapping);
-
-	tal_resize(&result->wrapping, indent+1);
-	result->wrapping[indent] = type;
-}
-
-static void result_pop_wrap(struct json_result *result, jsmntype_t type)
-{
-	size_t indent = tal_count(result->wrapping);
-
-	assert(indent);
-	assert(result->wrapping[indent-1] == type);
-	tal_resize(&result->wrapping, indent-1);
-}
-
-void json_array_start(struct json_result *result, const char *fieldname)
-{
-	json_start_member(result, fieldname);
-	result_append(result, "[");
-	result_add_wrap(result, JSMN_ARRAY);
-}
-
-void json_array_end(struct json_result *result)
-{
-	result_append(result, "\n");
-	result_pop_wrap(result, JSMN_ARRAY);
-	result_add_indent(result);
-	result_append(result, "]");
-}
-
-void json_object_start(struct json_result *result, const char *fieldname)
-{
-	json_start_member(result, fieldname);
-	result_append(result, "{");
-	result_add_wrap(result, JSMN_OBJECT);
-}
-
-void json_object_end(struct json_result *result)
-{
-	result_append(result, "\n");
-	result_pop_wrap(result, JSMN_OBJECT);
-	result_add_indent(result);
-	result_append(result, "}");
-}
-
-void json_add_num(struct json_result *result, const char *fieldname, unsigned int value)
-{
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "%u", value);
-}
-
-void json_add_double(struct json_result *result, const char *fieldname, double value)
-{
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "%f", value);
-}
-
-void json_add_u64(struct json_result *result, const char *fieldname,
-		  uint64_t value)
-{
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "%"PRIu64, value);
-}
-
-void json_add_literal(struct json_result *result, const char *fieldname,
-		      const char *literal, int len)
-{
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "%.*s", len, literal);
-}
-
-void json_add_string(struct json_result *result, const char *fieldname, const char *value)
-{
-	struct json_escaped *esc = json_partial_escape(NULL, value);
-
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "\"%s\"", esc->s);
-	tal_free(esc);
-}
-
-void json_add_bool(struct json_result *result, const char *fieldname, bool value)
-{
-	json_start_member(result, fieldname);
-	result_append(result, value ? "true" : "false");
-}
-
-void json_add_hex(struct json_result *result, const char *fieldname,
-		  const void *data, size_t len)
-{
-	char *hex = tal_arr(NULL, char, hex_str_size(len));
-
-	hex_encode(data, len, hex, hex_str_size(len));
-	json_add_string(result, fieldname, hex);
-	tal_free(hex);
-}
-
-void json_add_hex_talarr(struct json_result *result,
-			 const char *fieldname,
-			 const tal_t *data)
-{
-	json_add_hex(result, fieldname, data, tal_bytelen(data));
-}
-
-void json_add_object(struct json_result *result, ...)
-{
-	va_list ap;
-	const char *field;
-
-	va_start(ap, result);
-	json_object_start(result, NULL);
-	while ((field = va_arg(ap, const char *)) != NULL) {
-		jsmntype_t type = va_arg(ap, jsmntype_t);
-		const char *value = va_arg(ap, const char *);
-		if (type == JSMN_STRING)
-			json_add_string(result, field, value);
-		else
-			json_add_literal(result, field, value, strlen(value));
-	}
-	json_object_end(result);
-	va_end(ap);
-}
-
-void json_add_escaped_string(struct json_result *result, const char *fieldname,
-			     const struct json_escaped *esc TAKES)
-{
-	json_start_member(result, fieldname);
-	result_append_fmt(result, "\"%s\"", esc->s);
-	if (taken(esc))
-		tal_free(esc);
-}
-
-struct json_result *new_json_result(const tal_t *ctx)
-{
-	struct json_result *r = tal(ctx, struct json_result);
-
-	/* Using tal_arr means that it has a valid count. */
-	r->s = tal_arrz(r, char, 1);
-	r->wrapping = tal_arr(r, jsmntype_t, 0);
-	return r;
-}
-
-const char *json_result_string(const struct json_result *result)
-{
-	assert(tal_count(result->wrapping) == 0);
-	assert(tal_count(result->s) == strlen(result->s) + 1);
-	return result->s;
-}

common/json.h

@@ -12,7 +12,6 @@
 # include <external/jsmn/jsmn.h>
 
 struct json_escaped;
-struct json_result;
 struct short_channel_id;
 
 /* Include " if it's a string. */
@@ -58,53 +57,4 @@ const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index);
 /* If input is complete and valid, return tokens. */
 jsmntok_t *json_parse_input(const char *input, int len, bool *valid);
 
-/* Creating JSON strings */
-
-/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */
-void json_array_start(struct json_result *ptr, const char *fieldname);
-/* '"fieldname" : { ' or '{ ' if fieldname is NULL */
-void json_object_start(struct json_result *ptr, const char *fieldname);
-/* ' ], ' */
-void json_array_end(struct json_result *ptr);
-/* ' }, ' */
-void json_object_end(struct json_result *ptr);
-
-struct json_result *new_json_result(const tal_t *ctx);
-
-/* '"fieldname" : "value"' or '"value"' if fieldname is NULL.  Turns
- * any non-printable chars into JSON escapes, but leaves existing escapes alone.
- */
-void json_add_string(struct json_result *result, const char *fieldname, const char *value);
-
-/* '"fieldname" : "value"' or '"value"' if fieldname is NULL.  String must
- * already be JSON escaped as necessary. */
-void json_add_escaped_string(struct json_result *result,
-			     const char *fieldname,
-			     const struct json_escaped *esc TAKES);
-
-/* '"fieldname" : literal' or 'literal' if fieldname is NULL*/
-void json_add_literal(struct json_result *result, const char *fieldname,
-		      const char *literal, int len);
-/* '"fieldname" : value' or 'value' if fieldname is NULL */
-void json_add_double(struct json_result *result, const char *fieldname,
-		     double value);
-/* '"fieldname" : value' or 'value' if fieldname is NULL */
-void json_add_num(struct json_result *result, const char *fieldname,
-		  unsigned int value);
-/* '"fieldname" : value' or 'value' if fieldname is NULL */
-void json_add_u64(struct json_result *result, const char *fieldname,
-		  uint64_t value);
-/* '"fieldname" : true|false' or 'true|false' if fieldname is NULL */
-void json_add_bool(struct json_result *result, const char *fieldname,
-		   bool value);
-/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */
-void json_add_hex(struct json_result *result, const char *fieldname,
-		  const void *data, size_t len);
-/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */
-void json_add_hex_talarr(struct json_result *result,
-			 const char *fieldname,
-			 const tal_t *data);
-void json_add_object(struct json_result *result, ...);
-
-const char *json_result_string(const struct json_result *result);
 #endif /* LIGHTNING_COMMON_JSON_H */

common/test/run-json.c

@@ -1,5 +1,4 @@
 #include "../json.c"
-#include "../json_escaped.c"
 #include <common/utils.h>
 #include <stdio.h>
 
@@ -39,152 +38,11 @@ static int test_json_tok_bitcoin_amount(void)
 	return 0;
 }
 
-static int test_json_filter(void)
-{
-	struct json_result *result = new_json_result(NULL);
-	jsmntok_t *toks;
-	const jsmntok_t *x;
-	bool valid;
-	int i;
-	char *badstr = tal_arr(result, char, 256);
-	const char *str;
-
-	/* Fill with junk, and nul-terminate (256 -> 0) */
-	for (i = 1; i < 257; i++)
-		badstr[i-1] = i;
-
-	json_object_start(result, NULL);
-	json_add_string(result, "x", badstr);
-	json_object_end(result);
-
-	/* Parse back in, make sure nothing crazy. */
-	str = json_result_string(result);
-
-	toks = json_parse_input(str, strlen(str), &valid);
-	assert(valid);
-	assert(toks);
-
-	assert(toks[0].type == JSMN_OBJECT);
-	x = json_get_member(str, toks, "x");
-	assert(x);
-	assert(x->type == JSMN_STRING);
-
-	/* There are 7 one-letter escapes, and (32-5) \uXXXX escapes. */
-	assert((x->end - x->start) == 255 + 7*1 + (32-5)*5);
-	/* No control characters. */
-	for (i = x->start; i < x->end; i++) {
-		assert((unsigned)str[i] >= ' ');
-		assert((unsigned)str[i] != 127);
-	}
-	tal_free(result);
-	return 0;
-}
-
-static void test_json_escape(void)
-{
-	int i;
-
-	for (i = 1; i < 256; i++) {
-		char badstr[2];
-		struct json_result *result = new_json_result(NULL);
-		struct json_escaped *esc;
-
-		badstr[0] = i;
-		badstr[1] = 0;
-
-		json_object_start(result, NULL);
-		esc = json_escape(NULL, badstr);
-		json_add_escaped_string(result, "x", take(esc));
-		json_object_end(result);
-
-		const char *str = json_result_string(result);
-		if (i == '\\' || i == '"'
-		    || i == '\n' || i == '\r' || i == '\b'
-		    || i == '\t' || i == '\f')
-			assert(strstarts(str, "\n{\n  \"x\": \"\\"));
-		else if (i < 32 || i == 127) {
-			assert(strstarts(str, "\n{\n  \"x\": \"\\u00"));
-		} else {
-			char expect[] = "\n{\n  \"x\": \"?\"\n}";
-			expect[11] = i;
-			assert(streq(str, expect));
-		}
-		tal_free(result);
-	}
-}
-
-static void test_json_partial(void)
-{
-	const tal_t *ctx = tal(NULL, char);
-
-	assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\"));
-	assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\"));
-	assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\"));
-	assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\"));
-	assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n"));
-	assert(streq(json_partial_escape(ctx, "\n")->s, "\\n"));
-	assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\""));
-	assert(streq(json_partial_escape(ctx, "\"")->s, "\\\""));
-	assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t"));
-	assert(streq(json_partial_escape(ctx, "\t")->s, "\\t"));
-	assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b"));
-	assert(streq(json_partial_escape(ctx, "\b")->s, "\\b"));
-	assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r"));
-	assert(streq(json_partial_escape(ctx, "\r")->s, "\\r"));
-	assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f"));
-	assert(streq(json_partial_escape(ctx, "\f")->s, "\\f"));
-	/* You're allowed to escape / according to json.org. */
-	assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/"));
-	assert(streq(json_partial_escape(ctx, "/")->s, "/"));
-
-	assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF"));
-	assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx"));
-
-	/* Unknown escapes should be escaped. */
-	assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x"));
-	tal_free(ctx);
-}
-
-/* Test that we can segment and parse a stream of json objects correctly. */
-static void test_json_stream(void)
-{
-	bool valid;
-	char *input, *talstr;
-	jsmntok_t *toks;
-
-	/* Multiple full messages in a single buffer (happens when buffer
-	 * boundary coincides with message boundary, or read returned after
-	 * timeout. */
-	input = "{\"x\":\"x\"}{\"y\":\"y\"}";
-	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, strlen(talstr), &valid);
-	assert(toks);
-	assert(tal_count(toks) == 4);
-	assert(toks[0].start == 0 && toks[0].end == 9);
-	assert(valid);
-	tal_free(talstr);
-
-	/* Multiple messages, and the last one is partial, far more likely than
-	 * accidentally getting the boundaries to match. */
-	input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
-	talstr = tal_strndup(NULL, input, strlen(input));
-	toks = json_parse_input(talstr, strlen(talstr), &valid);
-	assert(toks);
-	assert(tal_count(toks) == 4);
-	assert(toks[0].start == 0 && toks[0].end == 9);
-	assert(valid);
-	tal_free(talstr);
-}
-
 int main(void)
 {
 	setup_locale();
 
 	test_json_tok_bitcoin_amount();
-	test_json_filter();
-	test_json_escape();
-	test_json_partial();
-	test_json_stream();
 	assert(!taken_any());
 	take_cleanup();
 }

lightningd/Makefile

@@ -34,7 +34,6 @@ LIGHTNINGD_COMMON_OBJS :=			\
 	common/key_derive.o			\
 	common/io_lock.o			\
 	common/json.o				\
-	common/json_escaped.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\
 	common/permute_tx.o			\
@@ -65,6 +64,7 @@ LIGHTNINGD_SRC :=				\
 	lightningd/htlc_end.c			\
 	lightningd/invoice.c			\
 	lightningd/json.c			\
+	lightningd/json_escaped.c		\
 	lightningd/jsonrpc.c			\
 	lightningd/lightningd.c			\
 	lightningd/log.c			\

lightningd/gossip_control.c

@@ -11,7 +11,6 @@
 #include <ccan/take/take.h>
 #include <ccan/tal/str/str.h>
 #include <common/features.h>
-#include <common/json_escaped.h>
 #include <common/type_to_string.h>
 #include <common/utils.h>
 #include <errno.h>
@@ -23,6 +22,7 @@
 #include <lightningd/gossip_msg.h>
 #include <lightningd/hsm_control.h>
 #include <lightningd/json.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/log.h>

lightningd/invoice.c

@@ -10,7 +10,6 @@
 #include <ccan/tal/str/str.h>
 #include <common/bech32.h>
 #include <common/bolt11.h>
-#include <common/json_escaped.h>
 #include <common/pseudorand.h>
 #include <common/utils.h>
 #include <errno.h>
@@ -19,6 +18,7 @@
 #include <inttypes.h>
 #include <lightningd/channel.h>
 #include <lightningd/hsm_control.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/log.h>
 #include <lightningd/options.h>

lightningd/json.c

@@ -7,6 +7,7 @@
 #include <common/type_to_string.h>
 #include <common/wireaddr.h>
 #include <gossipd/routing.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/options.h>
@@ -477,3 +478,239 @@ bool json_tok_tok(struct command *cmd, const char *name,
 {
 	return (*out = tok);
 }
+
+struct json_result {
+	/* tal_arr of types we're enclosed in. */
+	jsmntype_t *wrapping;
+
+	/* tal_count() of this is strlen() + 1 */
+	char *s;
+};
+
+static void result_append(struct json_result *res, const char *str)
+{
+	size_t len = tal_count(res->s) - 1;
+
+	tal_resize(&res->s, len + strlen(str) + 1);
+	strcpy(res->s + len, str);
+}
+
+static void PRINTF_FMT(2,3)
+result_append_fmt(struct json_result *res, const char *fmt, ...)
+{
+	size_t len = tal_count(res->s) - 1, fmtlen;
+	va_list ap;
+
+	va_start(ap, fmt);
+	fmtlen = vsnprintf(NULL, 0, fmt, ap);
+	va_end(ap);
+
+	tal_resize(&res->s, len + fmtlen + 1);
+	va_start(ap, fmt);
+	vsprintf(res->s + len, fmt, ap);
+	va_end(ap);
+}
+
+static bool result_ends_with(struct json_result *res, const char *str)
+{
+	size_t len = tal_count(res->s) - 1;
+
+	if (strlen(str) > len)
+		return false;
+	return streq(res->s + len - strlen(str), str);
+}
+
+static void check_fieldname(const struct json_result *result,
+			    const char *fieldname)
+{
+	size_t n = tal_count(result->wrapping);
+	if (n == 0)
+		/* Can't have a fieldname if not in anything! */
+		assert(!fieldname);
+	else if (result->wrapping[n-1] == JSMN_ARRAY)
+		/* No fieldnames in arrays. */
+		assert(!fieldname);
+	else
+		/* Must have fieldnames in objects. */
+		assert(fieldname);
+}
+
+static void result_add_indent(struct json_result *result);
+
+	static void json_start_member(struct json_result *result, const char *fieldname)
+{
+	/* Prepend comma if required. */
+	if (result->s[0]
+	    && !result_ends_with(result, "{")
+	    && !result_ends_with(result, "["))
+		result_append(result, ", \n");
+	else
+		result_append(result, "\n");
+
+	result_add_indent(result);
+
+	check_fieldname(result, fieldname);
+	if (fieldname)
+		result_append_fmt(result, "\"%s\": ", fieldname);
+}
+
+static void result_add_indent(struct json_result *result)
+{
+	size_t i, indent = tal_count(result->wrapping);
+
+	if (!indent)
+		return;
+
+	for (i = 0; i < indent; i++)
+		result_append(result, "  ");
+}
+
+static void result_add_wrap(struct json_result *result, jsmntype_t type)
+{
+	size_t indent = tal_count(result->wrapping);
+
+	tal_resize(&result->wrapping, indent+1);
+	result->wrapping[indent] = type;
+}
+
+static void result_pop_wrap(struct json_result *result, jsmntype_t type)
+{
+	size_t indent = tal_count(result->wrapping);
+
+	assert(indent);
+	assert(result->wrapping[indent-1] == type);
+	tal_resize(&result->wrapping, indent-1);
+}
+
+void json_array_start(struct json_result *result, const char *fieldname)
+{
+	json_start_member(result, fieldname);
+	result_append(result, "[");
+	result_add_wrap(result, JSMN_ARRAY);
+}
+
+void json_array_end(struct json_result *result)
+{
+	result_append(result, "\n");
+	result_pop_wrap(result, JSMN_ARRAY);
+	result_add_indent(result);
+	result_append(result, "]");
+}
+
+void json_object_start(struct json_result *result, const char *fieldname)
+{
+	json_start_member(result, fieldname);
+	result_append(result, "{");
+	result_add_wrap(result, JSMN_OBJECT);
+}
+
+void json_object_end(struct json_result *result)
+{
+	result_append(result, "\n");
+	result_pop_wrap(result, JSMN_OBJECT);
+	result_add_indent(result);
+	result_append(result, "}");
+}
+
+void json_add_num(struct json_result *result, const char *fieldname, unsigned int value)
+{
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "%u", value);
+}
+
+void json_add_double(struct json_result *result, const char *fieldname, double value)
+{
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "%f", value);
+}
+
+void json_add_u64(struct json_result *result, const char *fieldname,
+		  uint64_t value)
+{
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "%"PRIu64, value);
+}
+
+void json_add_literal(struct json_result *result, const char *fieldname,
+		      const char *literal, int len)
+{
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "%.*s", len, literal);
+}
+
+void json_add_string(struct json_result *result, const char *fieldname, const char *value)
+{
+	struct json_escaped *esc = json_partial_escape(NULL, value);
+
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "\"%s\"", esc->s);
+	tal_free(esc);
+}
+
+void json_add_bool(struct json_result *result, const char *fieldname, bool value)
+{
+	json_start_member(result, fieldname);
+	result_append(result, value ? "true" : "false");
+}
+
+void json_add_hex(struct json_result *result, const char *fieldname,
+		  const void *data, size_t len)
+{
+	char *hex = tal_arr(NULL, char, hex_str_size(len));
+
+	hex_encode(data, len, hex, hex_str_size(len));
+	json_add_string(result, fieldname, hex);
+	tal_free(hex);
+}
+
+void json_add_hex_talarr(struct json_result *result,
+			 const char *fieldname,
+			 const tal_t *data)
+{
+	json_add_hex(result, fieldname, data, tal_bytelen(data));
+}
+
+void json_add_object(struct json_result *result, ...)
+{
+	va_list ap;
+	const char *field;
+
+	va_start(ap, result);
+	json_object_start(result, NULL);
+	while ((field = va_arg(ap, const char *)) != NULL) {
+		jsmntype_t type = va_arg(ap, jsmntype_t);
+		const char *value = va_arg(ap, const char *);
+		if (type == JSMN_STRING)
+			json_add_string(result, field, value);
+		else
+			json_add_literal(result, field, value, strlen(value));
+	}
+	json_object_end(result);
+	va_end(ap);
+}
+
+void json_add_escaped_string(struct json_result *result, const char *fieldname,
+			     const struct json_escaped *esc TAKES)
+{
+	json_start_member(result, fieldname);
+	result_append_fmt(result, "\"%s\"", esc->s);
+	if (taken(esc))
+		tal_free(esc);
+}
+
+struct json_result *new_json_result(const tal_t *ctx)
+{
+	struct json_result *r = tal(ctx, struct json_result);
+
+	/* Using tal_arr means that it has a valid count. */
+	r->s = tal_arrz(r, char, 1);
+	r->wrapping = tal_arr(r, jsmntype_t, 0);
+	return r;
+}
+
+const char *json_result_string(const struct json_result *result)
+{
+	assert(tal_count(result->wrapping) == 0);
+	assert(tal_count(result->s) == strlen(result->s) + 1);
+	return result->s;
+}

lightningd/json.h

@@ -6,6 +6,7 @@
 #define LIGHTNING_LIGHTNINGD_JSON_H
 #include "config.h"
 #include <ccan/short_types/short_types.h>
+#include <ccan/tal/tal.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
@@ -157,4 +158,54 @@ bool json_tok_tok(struct command *cmd, const char *name,
 		  const char *buffer, const jsmntok_t * tok,
 		  const jsmntok_t **out);
 
+
+/* Creating JSON strings */
+
+/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */
+void json_array_start(struct json_result *ptr, const char *fieldname);
+/* '"fieldname" : { ' or '{ ' if fieldname is NULL */
+void json_object_start(struct json_result *ptr, const char *fieldname);
+/* ' ], ' */
+void json_array_end(struct json_result *ptr);
+/* ' }, ' */
+void json_object_end(struct json_result *ptr);
+
+struct json_result *new_json_result(const tal_t *ctx);
+
+/* '"fieldname" : "value"' or '"value"' if fieldname is NULL.  Turns
+ * any non-printable chars into JSON escapes, but leaves existing escapes alone.
+ */
+void json_add_string(struct json_result *result, const char *fieldname, const char *value);
+
+/* '"fieldname" : "value"' or '"value"' if fieldname is NULL.  String must
+ * already be JSON escaped as necessary. */
+void json_add_escaped_string(struct json_result *result,
+			     const char *fieldname,
+			     const struct json_escaped *esc TAKES);
+
+/* '"fieldname" : literal' or 'literal' if fieldname is NULL*/
+void json_add_literal(struct json_result *result, const char *fieldname,
+		      const char *literal, int len);
+/* '"fieldname" : value' or 'value' if fieldname is NULL */
+void json_add_double(struct json_result *result, const char *fieldname,
+		     double value);
+/* '"fieldname" : value' or 'value' if fieldname is NULL */
+void json_add_num(struct json_result *result, const char *fieldname,
+		  unsigned int value);
+/* '"fieldname" : value' or 'value' if fieldname is NULL */
+void json_add_u64(struct json_result *result, const char *fieldname,
+		  uint64_t value);
+/* '"fieldname" : true|false' or 'true|false' if fieldname is NULL */
+void json_add_bool(struct json_result *result, const char *fieldname,
+		   bool value);
+/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */
+void json_add_hex(struct json_result *result, const char *fieldname,
+		  const void *data, size_t len);
+/* '"fieldname" : "0189abcdef..."' or "0189abcdef..." if fieldname is NULL */
+void json_add_hex_talarr(struct json_result *result,
+			 const char *fieldname,
+			 const tal_t *data);
+void json_add_object(struct json_result *result, ...);
+
+const char *json_result_string(const struct json_result *result);
 #endif /* LIGHTNING_LIGHTNINGD_JSON_H */

common/json_escaped.c → lightningd/json_escaped.c

@@ -1,4 +1,4 @@
-#include <common/json_escaped.h>
+#include <lightningd/json_escaped.h>
 #include <stdio.h>
 
 struct json_escaped *json_escaped_string_(const tal_t *ctx,

common/json_escaped.h → lightningd/json_escaped.h

@@ -1,5 +1,5 @@
-#ifndef LIGHTNING_COMMON_JSON_ESCAPED_H
+#ifndef LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H
-#define LIGHTNING_COMMON_JSON_ESCAPED_H
+#define LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H
 #include "config.h"
 #include <common/json.h>
 
@@ -32,4 +32,4 @@ struct json_escaped *json_escaped_string_(const tal_t *ctx,
 /* Be very careful here!  Can fail!  Doesn't handle \u: use UTF-8 please. */
 const char *json_escaped_unescape(const tal_t *ctx,
 				  const struct json_escaped *esc);
-#endif /* LIGHTNING_COMMON_JSON_ESCAPED_H */
+#endif /* LIGHTNING_LIGHTNINGD_JSON_ESCAPED_H */

lightningd/jsonrpc.c

@@ -10,7 +10,6 @@
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
 #include <common/bech32.h>
-#include <common/json_escaped.h>
 #include <common/memleak.h>
 #include <common/version.h>
 #include <common/wallet_tx.h>
@@ -19,6 +18,7 @@
 #include <fcntl.h>
 #include <lightningd/chaintopology.h>
 #include <lightningd/json.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/lightningd.h>

lightningd/lightningd.c

@@ -69,6 +69,7 @@
 #include <lightningd/channel_control.h>
 #include <lightningd/connect_control.h>
 #include <lightningd/invoice.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/log.h>
 #include <lightningd/onchain_control.h>

lightningd/lightningd.h

@@ -6,7 +6,6 @@
 #include <ccan/container_of/container_of.h>
 #include <ccan/time/time.h>
 #include <ccan/timer/timer.h>
-#include <common/json_escaped.h>
 #include <lightningd/htlc_end.h>
 #include <stdio.h>
 #include <wallet/txfilter.h>

lightningd/options.c

@@ -10,7 +10,6 @@
 #include <ccan/tal/path/path.h>
 #include <ccan/tal/str/str.h>
 #include <common/configdir.h>
-#include <common/json_escaped.h>
 #include <common/memleak.h>
 #include <common/version.h>
 #include <common/wireaddr.h>
@@ -20,6 +19,7 @@
 #include <lightningd/bitcoind.h>
 #include <lightningd/chaintopology.h>
 #include <lightningd/json.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/lightningd.h>

lightningd/peer_control.c

@@ -15,7 +15,6 @@
 #include <common/dev_disconnect.h>
 #include <common/features.h>
 #include <common/initial_commit_tx.h>
-#include <common/json_escaped.h>
 #include <common/key_derive.h>
 #include <common/status.h>
 #include <common/timeout.h>

lightningd/peer_htlcs.c

@@ -5,7 +5,6 @@
 #include <ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
 #include <channeld/gen_channel_wire.h>
-#include <common/json_escaped.h>
 #include <common/overflows.h>
 #include <common/sphinx.h>
 #include <common/timeout.h>
@@ -13,6 +12,7 @@
 #include <lightningd/chaintopology.h>
 #include <lightningd/htlc_end.h>
 #include <lightningd/json.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/jsonrpc.h>
 #include <lightningd/jsonrpc_errors.h>
 #include <lightningd/lightningd.h>

lightningd/test/Makefile

@@ -11,6 +11,7 @@ LIGHTNINGD_TEST_COMMON_OBJS :=			\
 	common/bech32.o				\
 	common/daemon_conn.o			\
 	common/htlc_state.o			\
+	common/json.o				\
 	common/pseudorand.o			\
 	common/memleak.o			\
 	common/msg_queue.o			\

lightningd/test/run-invoice-select-inchan.c

@@ -156,9 +156,6 @@ void json_array_start(struct json_result *ptr UNNEEDED, const char *fieldname UN
 /* Generated stub for json_escape */
 struct json_escaped *json_escape(const tal_t *ctx UNNEEDED, const char *str TAKES UNNEEDED)
 { fprintf(stderr, "json_escape called!\n"); abort(); }
-/* Generated stub for json_next */
-const jsmntok_t *json_next(const jsmntok_t *tok UNNEEDED)
-{ fprintf(stderr, "json_next called!\n"); abort(); }
 /* Generated stub for json_object_end */
 void json_object_end(struct json_result *ptr UNNEEDED)
 { fprintf(stderr, "json_object_end called!\n"); abort(); }

lightningd/test/run-json_escaped.c

@@ -0,0 +1,46 @@
+#include "../json_escaped.c"
+#include <assert.h>
+#include <common/utils.h>
+#include <stdio.h>
+
+static void test_json_partial(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+
+	assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n"));
+	assert(streq(json_partial_escape(ctx, "\n")->s, "\\n"));
+	assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\""));
+	assert(streq(json_partial_escape(ctx, "\"")->s, "\\\""));
+	assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t"));
+	assert(streq(json_partial_escape(ctx, "\t")->s, "\\t"));
+	assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b"));
+	assert(streq(json_partial_escape(ctx, "\b")->s, "\\b"));
+	assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r"));
+	assert(streq(json_partial_escape(ctx, "\r")->s, "\\r"));
+	assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f"));
+	assert(streq(json_partial_escape(ctx, "\f")->s, "\\f"));
+	/* You're allowed to escape / according to json.org. */
+	assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/"));
+	assert(streq(json_partial_escape(ctx, "/")->s, "/"));
+
+	assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF"));
+	assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx"));
+
+	/* Unknown escapes should be escaped. */
+	assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x"));
+	tal_free(ctx);
+}
+
+
+int main(void)
+{
+	setup_locale();
+
+	test_json_partial();
+	assert(!taken_any());
+	take_cleanup();
+}

lightningd/test/run-jsonrpc.c

@@ -0,0 +1,218 @@
+#include "../json_escaped.c"
+#include "../jsonrpc.c"
+#include "../json.c"
+
+/* AUTOGENERATED MOCKS START */
+/* Generated stub for db_begin_transaction_ */
+void db_begin_transaction_(struct db *db UNNEEDED, const char *location UNNEEDED)
+{ fprintf(stderr, "db_begin_transaction_ called!\n"); abort(); }
+/* Generated stub for db_commit_transaction */
+void db_commit_transaction(struct db *db UNNEEDED)
+{ fprintf(stderr, "db_commit_transaction called!\n"); abort(); }
+/* Generated stub for fatal */
+void   fatal(const char *fmt UNNEEDED, ...)
+{ fprintf(stderr, "fatal called!\n"); abort(); }
+/* Generated stub for feerate_from_style */
+u32 feerate_from_style(u32 feerate UNNEEDED, enum feerate_style style UNNEEDED)
+{ fprintf(stderr, "feerate_from_style called!\n"); abort(); }
+/* Generated stub for feerate_name */
+const char *feerate_name(enum feerate feerate UNNEEDED)
+{ fprintf(stderr, "feerate_name called!\n"); abort(); }
+/* Generated stub for fmt_wireaddr_without_port */
+char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED)
+{ fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); }
+/* Generated stub for get_block_height */
+u32 get_block_height(const struct chain_topology *topo UNNEEDED)
+{ fprintf(stderr, "get_block_height called!\n"); abort(); }
+/* Generated stub for get_chainparams */
+const struct chainparams *get_chainparams(const struct lightningd *ld UNNEEDED)
+{ fprintf(stderr, "get_chainparams called!\n"); abort(); }
+/* Generated stub for io_lock_acquire_out_ */
+struct io_plan *io_lock_acquire_out_(struct io_conn *conn UNNEEDED, struct io_lock *lock UNNEEDED,
+				     struct io_plan *(*next)(struct io_conn * UNNEEDED,
+							     void *) UNNEEDED,
+				     void *arg UNNEEDED)
+{ fprintf(stderr, "io_lock_acquire_out_ called!\n"); abort(); }
+/* Generated stub for io_lock_new */
+struct io_lock *io_lock_new(const tal_t *ctx UNNEEDED)
+{ fprintf(stderr, "io_lock_new called!\n"); abort(); }
+/* Generated stub for io_lock_release */
+void io_lock_release(struct io_lock *lock UNNEEDED)
+{ fprintf(stderr, "io_lock_release called!\n"); abort(); }
+/* Generated stub for json_feerate_estimate */
+bool json_feerate_estimate(struct command *cmd UNNEEDED,
+			   u32 **feerate_per_kw UNNEEDED, enum feerate feerate UNNEEDED)
+{ fprintf(stderr, "json_feerate_estimate called!\n"); abort(); }
+/* Generated stub for log_ */
+void log_(struct log *log UNNEEDED, enum log_level level UNNEEDED, const char *fmt UNNEEDED, ...)
+
+{ fprintf(stderr, "log_ called!\n"); abort(); }
+/* Generated stub for log_io */
+void log_io(struct log *log UNNEEDED, enum log_level dir UNNEEDED, const char *comment UNNEEDED,
+	    const void *data UNNEEDED, size_t len UNNEEDED)
+{ fprintf(stderr, "log_io called!\n"); abort(); }
+/* Generated stub for log_prefix */
+const char *log_prefix(const struct log *log UNNEEDED)
+{ fprintf(stderr, "log_prefix called!\n"); abort(); }
+/* Generated stub for new_log */
+struct log *new_log(const tal_t *ctx UNNEEDED, struct log_book *record UNNEEDED, const char *fmt UNNEEDED, ...)
+{ fprintf(stderr, "new_log called!\n"); abort(); }
+/* Generated stub for param */
+bool param(struct command *cmd UNNEEDED, const char *buffer UNNEEDED,
+	   const jsmntok_t params[] UNNEEDED, ...)
+{ fprintf(stderr, "param called!\n"); abort(); }
+/* Generated stub for version */
+const char *version(void)
+{ fprintf(stderr, "version called!\n"); abort(); }
+/* AUTOGENERATED MOCKS END */
+
+bool deprecated_apis;
+
+static int test_json_filter(void)
+{
+	struct json_result *result = new_json_result(NULL);
+	jsmntok_t *toks;
+	const jsmntok_t *x;
+	bool valid;
+	int i;
+	char *badstr = tal_arr(result, char, 256);
+	const char *str;
+
+	/* Fill with junk, and nul-terminate (256 -> 0) */
+	for (i = 1; i < 257; i++)
+		badstr[i-1] = i;
+
+	json_object_start(result, NULL);
+	json_add_string(result, "x", badstr);
+	json_object_end(result);
+
+	/* Parse back in, make sure nothing crazy. */
+	str = json_result_string(result);
+
+	toks = json_parse_input(str, strlen(str), &valid);
+	assert(valid);
+	assert(toks);
+
+	assert(toks[0].type == JSMN_OBJECT);
+	x = json_get_member(str, toks, "x");
+	assert(x);
+	assert(x->type == JSMN_STRING);
+
+	/* There are 7 one-letter escapes, and (32-5) \uXXXX escapes. */
+	assert((x->end - x->start) == 255 + 7*1 + (32-5)*5);
+	/* No control characters. */
+	for (i = x->start; i < x->end; i++) {
+		assert((unsigned)str[i] >= ' ');
+		assert((unsigned)str[i] != 127);
+	}
+	tal_free(result);
+	return 0;
+}
+
+static void test_json_escape(void)
+{
+	int i;
+
+	for (i = 1; i < 256; i++) {
+		char badstr[2];
+		struct json_result *result = new_json_result(NULL);
+		struct json_escaped *esc;
+
+		badstr[0] = i;
+		badstr[1] = 0;
+
+		json_object_start(result, NULL);
+		esc = json_escape(NULL, badstr);
+		json_add_escaped_string(result, "x", take(esc));
+		json_object_end(result);
+
+		const char *str = json_result_string(result);
+		if (i == '\\' || i == '"'
+		    || i == '\n' || i == '\r' || i == '\b'
+		    || i == '\t' || i == '\f')
+			assert(strstarts(str, "\n{\n  \"x\": \"\\"));
+		else if (i < 32 || i == 127) {
+			assert(strstarts(str, "\n{\n  \"x\": \"\\u00"));
+		} else {
+			char expect[] = "\n{\n  \"x\": \"?\"\n}";
+			expect[11] = i;
+			assert(streq(str, expect));
+		}
+		tal_free(result);
+	}
+}
+
+static void test_json_partial(void)
+{
+	const tal_t *ctx = tal(NULL, char);
+
+	assert(streq(json_partial_escape(ctx, "\\")->s, "\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\")->s, "\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\\\")->s, "\\\\\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\\\\\\\")->s, "\\\\\\\\"));
+	assert(streq(json_partial_escape(ctx, "\\n")->s, "\\n"));
+	assert(streq(json_partial_escape(ctx, "\n")->s, "\\n"));
+	assert(streq(json_partial_escape(ctx, "\\\"")->s, "\\\""));
+	assert(streq(json_partial_escape(ctx, "\"")->s, "\\\""));
+	assert(streq(json_partial_escape(ctx, "\\t")->s, "\\t"));
+	assert(streq(json_partial_escape(ctx, "\t")->s, "\\t"));
+	assert(streq(json_partial_escape(ctx, "\\b")->s, "\\b"));
+	assert(streq(json_partial_escape(ctx, "\b")->s, "\\b"));
+	assert(streq(json_partial_escape(ctx, "\\r")->s, "\\r"));
+	assert(streq(json_partial_escape(ctx, "\r")->s, "\\r"));
+	assert(streq(json_partial_escape(ctx, "\\f")->s, "\\f"));
+	assert(streq(json_partial_escape(ctx, "\f")->s, "\\f"));
+	/* You're allowed to escape / according to json.org. */
+	assert(streq(json_partial_escape(ctx, "\\/")->s, "\\/"));
+	assert(streq(json_partial_escape(ctx, "/")->s, "/"));
+
+	assert(streq(json_partial_escape(ctx, "\\u0FFF")->s, "\\u0FFF"));
+	assert(streq(json_partial_escape(ctx, "\\u0FFFx")->s, "\\u0FFFx"));
+
+	/* Unknown escapes should be escaped. */
+	assert(streq(json_partial_escape(ctx, "\\x")->s, "\\\\x"));
+	tal_free(ctx);
+}
+
+/* Test that we can segment and parse a stream of json objects correctly. */
+static void test_json_stream(void)
+{
+	bool valid;
+	char *input, *talstr;
+	jsmntok_t *toks;
+
+	/* Multiple full messages in a single buffer (happens when buffer
+	 * boundary coincides with message boundary, or read returned after
+	 * timeout. */
+	input = "{\"x\":\"x\"}{\"y\":\"y\"}";
+	talstr = tal_strndup(NULL, input, strlen(input));
+	toks = json_parse_input(talstr, strlen(talstr), &valid);
+	assert(toks);
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	assert(valid);
+	tal_free(talstr);
+
+	/* Multiple messages, and the last one is partial, far more likely than
+	 * accidentally getting the boundaries to match. */
+	input = "{\"x\":\"x\"}{\"y\":\"y\"}{\"z\":\"z";
+	talstr = tal_strndup(NULL, input, strlen(input));
+	toks = json_parse_input(talstr, strlen(talstr), &valid);
+	assert(toks);
+	assert(tal_count(toks) == 4);
+	assert(toks[0].start == 0 && toks[0].end == 9);
+	assert(valid);
+	tal_free(talstr);
+}
+
+int main(void)
+{
+	setup_locale();
+
+	test_json_filter();
+	test_json_escape();
+	test_json_partial();
+	test_json_stream();
+	assert(!taken_any());
+	take_cleanup();
+}

lightningd/test/run-param.c

@@ -1,14 +1,12 @@
-#include "config.h"
+#include "../json.c"
-#include <signal.h>
+#include "../json_escaped.c"
-#include <setjmp.h>
+#include "../param.c"
-#include <lightningd/jsonrpc.h>
-
-#include <lightningd/json.c>
-#include <lightningd/param.c>
-#include <common/json.c>
-#include <common/json_escaped.c>
 #include <ccan/array_size/array_size.h>
 #include <ccan/err/err.h>
+#include <common/json.h>
+#include <lightningd/jsonrpc.h>
+#include <setjmp.h>
+#include <signal.h>
 #include <unistd.h>
 
 char *fail_msg = NULL;

wallet/db.c

@@ -1,9 +1,9 @@
 #include "db.h"
 
 #include <ccan/tal/str/str.h>
-#include <common/json_escaped.h>
 #include <common/version.h>
 #include <inttypes.h>
+#include <lightningd/json_escaped.h>
 #include <lightningd/lightningd.h>
 #include <lightningd/log.h>