common/json: make json_scan return an error string.

This makes for more useful errors.  It prints where it was up to in
the guide, but doesn't print the entire JSON it's scanning.

Suggested-by: Christian Decker
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/json.c

@@ -720,30 +720,42 @@ void json_tok_remove(jsmntok_t **tokens,
 }
 
 /* talfmt take a ctx pointer and return NULL or a valid pointer.
- * fmt takes the argument, and returns a bool. */
-static bool handle_percent(const char *buffer,
-			   const jsmntok_t *tok,
-			   va_list *ap)
+ * fmt takes the argument, and returns a bool.
+ *
+ * This function returns NULL on success, or errmsg on failure.
+*/
+static const char *handle_percent(const char *buffer,
+				  const jsmntok_t *tok,
+				  va_list *ap)
 {
 	void *ctx;
+	const char *fmtname;
 
 	/* This is set to (dummy) json_scan if it's a non-tal fmt */
 	ctx = va_arg(*ap, void *);
+	fmtname = va_arg(*ap, const char *);
 	if (ctx != json_scan) {
 		void *(*talfmt)(void *, const char *, const jsmntok_t *);
 		void **p;
 		p = va_arg(*ap, void **);
 		talfmt = va_arg(*ap, void *(*)(void *, const char *, const jsmntok_t *));
 		*p = talfmt(ctx, buffer, tok);
-		return *p != NULL;
+		if (*p != NULL)
+			return NULL;
 	} else {
 		bool (*fmt)(const char *, const jsmntok_t *, void *);
 		void *p;
 
 		p = va_arg(*ap, void *);
 		fmt = va_arg(*ap, bool (*)(const char *, const jsmntok_t *, void *));
-		return fmt(buffer, tok, p);
+		if (fmt(buffer, tok, p))
+			return NULL;
 	}
+
+	return tal_fmt(tmpctx, "%s could not parse %.*s",
+		       fmtname,
+		       json_tok_full_len(tok),
+		       json_tok_full(buffer, tok));
 }
 
 /* GUIDE := OBJ | ARRAY | '%'
@@ -756,235 +768,280 @@ static bool handle_percent(const char *buffer,
  * ARRELEM := NUMBER ':' FIELDVAL
  */
 
-/* Returns NULL on failure, or offset into guide */
-static const char *parse_literal(const char *guide,
-				 const char **literal,
-				 size_t *len)
+static void parse_literal(const char **guide,
+			  const char **literal,
+			  size_t *len)
 {
-	*literal = guide;
-	*len = strspn(guide,
+	*literal = *guide;
+	*len = strspn(*guide,
 		      "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 		      "abcdefghijklmnopqrstuvwxyz"
 		      "0123456789"
 		      "_-");
-	return guide + *len;
+	*guide += *len;
 }
 
-static const char *parse_number(const char *guide, u32 *number)
+static void parse_number(const char **guide, u32 *number)
 {
 	char *endp;
 	long int l;
 
-	l = strtol(guide, &endp, 10);
-	if (endp == guide || errno == ERANGE)
-		return NULL;
+	l = strtol(*guide, &endp, 10);
+	assert(endp != *guide);
+	assert(errno != ERANGE);
 
 	/* Test for overflow */
 	*number = l;
-	if (*number != l)
-		return NULL;
+	assert(*number == l);
 
-	return endp;
+	*guide = endp;
 }
 
-/* Recursion */
+static char guide_consume_one(const char **guide)
+{
+	char c = **guide;
+	(*guide)++;
+	return c;
+}
+
+static void guide_must_be(const char **guide, char c)
+{
+	char actual = guide_consume_one(guide);
+	assert(actual == c);
+}
+
+/* Recursion: return NULL on success, errmsg on fail */
 static const char *parse_obj(const char *buffer,
 			     const jsmntok_t *tok,
-			     const char *guide,
+			     const char **guide,
 			     va_list *ap);
 
 static const char *parse_arr(const char *buffer,
 			     const jsmntok_t *tok,
-			     const char *guide,
+			     const char **guide,
 			     va_list *ap);
 
 static const char *parse_guide(const char *buffer,
 			       const jsmntok_t *tok,
-			       const char *guide,
+			       const char **guide,
 			       va_list *ap)
 {
-	if (*guide == '{') {
-		guide = parse_obj(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		assert(*guide == '}');
-		return guide + 1;
-	} else if (*guide == '[') {
-		guide = parse_arr(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		assert(*guide == ']');
-		return guide + 1;
+	const char *errmsg;
+
+	if (**guide == '{') {
+		errmsg = parse_obj(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
+	} else if (**guide == '[') {
+		errmsg = parse_arr(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
 	} else {
-		assert(*guide == '%');
-		if (!handle_percent(buffer, tok, ap))
-			return NULL;
-		return guide + 1;
+		guide_must_be(guide, '%');
+		errmsg = handle_percent(buffer, tok, ap);
+		if (errmsg)
+			return errmsg;
 	}
+	return NULL;
 }
 
 static const char *parse_fieldval(const char *buffer,
 				  const jsmntok_t *tok,
-				  const char *guide,
+				  const char **guide,
 				  va_list *ap)
 {
-	if (*guide == '{') {
-		guide = parse_obj(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		assert(*guide == '}');
-		return guide + 1;
-	} else if (*guide == '[') {
-		guide = parse_arr(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		assert(*guide == ']');
-		return guide + 1;
-	} else if (*guide == '%') {
-		if (!handle_percent(buffer, tok, ap))
-			return NULL;
-		return guide + 1;
+	const char *errmsg;
+
+	if (**guide == '{') {
+		errmsg = parse_obj(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
+	} else if (**guide == '[') {
+		errmsg = parse_arr(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
+	} else if (**guide == '%') {
+		guide_consume_one(guide);
+		errmsg = handle_percent(buffer, tok, ap);
+		if (errmsg)
+			return errmsg;
 	} else {
 		const char *literal;
 		size_t len;
 
-		/* Literal must match exactly */
-		guide = parse_literal(guide, &literal, &len);
+		/* Literal must match exactly (modulo quotes for strings) */
+		parse_literal(guide, &literal, &len);
 		if (!memeq(buffer + tok->start, tok->end - tok->start,
-			   literal, len))
-			return NULL;
-		return guide;
+			   literal, len)) {
+			return tal_fmt(tmpctx,
+				       "%.*s does not match expected %.*s",
+				       json_tok_full_len(tok),
+				       json_tok_full(buffer, tok),
+				       (int)len, literal);
+		}
 	}
+	return NULL;
 }
 
 static const char *parse_field(const char *buffer,
 			       const jsmntok_t *tok,
-			       const char *guide,
+			       const char **guide,
 			       va_list *ap)
 {
 	const jsmntok_t *member;
 	size_t len;
 	const char *memname;
 
-	guide = parse_literal(guide, &memname, &len);
-	assert(*guide == ':');
+	parse_literal(guide, &memname, &len);
+	guide_must_be(guide, ':');
 
-	member = json_get_membern(buffer, tok, memname, guide - memname);
-	if (!member)
-		return NULL;
+	member = json_get_membern(buffer, tok, memname, len);
+	if (!member) {
+		return tal_fmt(tmpctx, "object does not have member %.*s",
+			       (int)len, memname);
+	}
 
-	return parse_fieldval(buffer, member, guide + 1, ap);
+	return parse_fieldval(buffer, member, guide, ap);
 }
 
 static const char *parse_fieldlist(const char *buffer,
 				   const jsmntok_t *tok,
-				   const char *guide,
+				   const char **guide,
 				   va_list *ap)
 {
 	for (;;) {
-		guide = parse_field(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		if (*guide != ',')
+		const char *errmsg;
+
+		errmsg = parse_field(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
+		if (**guide != ',')
 			break;
-		guide++;
+		guide_consume_one(guide);
 	}
-	return guide;
+	return NULL;
 }
 
 static const char *parse_obj(const char *buffer,
 			     const jsmntok_t *tok,
-			     const char *guide,
+			     const char **guide,
 			     va_list *ap)
 {
-	assert(*guide == '{');
+	const char *errmsg;
 
-	if (tok->type != JSMN_OBJECT)
-		return NULL;
+	guide_must_be(guide, '{');
 
-	guide = parse_fieldlist(buffer, tok, guide + 1, ap);
-	if (!guide)
-		return NULL;
-	return guide;
+	if (tok->type != JSMN_OBJECT) {
+		return tal_fmt(tmpctx, "token is not an object: %.*s",
+			       json_tok_full_len(tok),
+			       json_tok_full(buffer, tok));
+	}
+
+	errmsg = parse_fieldlist(buffer, tok, guide, ap);
+	if (errmsg)
+		return errmsg;
+
+	guide_must_be(guide, '}');
+	return NULL;
 }
 
 static const char *parse_arrelem(const char *buffer,
 				 const jsmntok_t *tok,
-				 const char *guide,
+				 const char **guide,
 				 va_list *ap)
 {
 	const jsmntok_t *member;
 	u32 idx;
 
-	guide = parse_number(guide, &idx);
-	assert(*guide == ':');
+	parse_number(guide, &idx);
+	guide_must_be(guide, ':');
 
 	member = json_get_arr(tok, idx);
-	if (!member)
-		return NULL;
+	if (!member) {
+		return tal_fmt(tmpctx, "token has no index %u: %.*s",
+			       idx,
+			       json_tok_full_len(tok),
+			       json_tok_full(buffer, tok));
+	}
 
-	return parse_fieldval(buffer, member, guide + 1, ap);
+	return parse_fieldval(buffer, member, guide, ap);
 }
 
 static const char *parse_arrlist(const char *buffer,
-				   const jsmntok_t *tok,
-				   const char *guide,
-				   va_list *ap)
+				 const jsmntok_t *tok,
+				 const char **guide,
+				 va_list *ap)
 {
+	const char *errmsg;
+
 	for (;;) {
-		guide = parse_arrelem(buffer, tok, guide, ap);
-		if (!guide)
-			return NULL;
-		if (*guide != ',')
+		errmsg = parse_arrelem(buffer, tok, guide, ap);
+		if (errmsg)
+			return errmsg;
+		if (**guide != ',')
 			break;
-		guide++;
+		guide_consume_one(guide);
 	}
-	return guide;
+	return NULL;
 }
+
 static const char *parse_arr(const char *buffer,
 			     const jsmntok_t *tok,
-			     const char *guide,
+			     const char **guide,
 			     va_list *ap)
 {
-	assert(*guide == '[');
+	const char *errmsg;
 
-	if (tok->type != JSMN_ARRAY)
-		return NULL;
+	guide_must_be(guide, '[');
 
-	guide = parse_arrlist(buffer, tok, guide + 1, ap);
-	if (!guide)
-		return NULL;
-	return guide;
+	if (tok->type != JSMN_ARRAY) {
+		return tal_fmt(tmpctx, "token is not an array: %.*s",
+			       json_tok_full_len(tok),
+			       json_tok_full(buffer, tok));
+	}
+
+	errmsg = parse_arrlist(buffer, tok, guide, ap);
+	if (errmsg)
+		return errmsg;
+
+	guide_must_be(guide, ']');
+	return NULL;
 }
 
-bool json_scanv(const char *buffer,
-		const jsmntok_t *tok,
-		const char *guide,
-		va_list ap)
+const char *json_scanv(const tal_t *ctx,
+		       const char *buffer,
+		       const jsmntok_t *tok,
+		       const char *guide,
+		       va_list ap)
 {
 	va_list cpy;
+	const char *orig_guide = guide, *errmsg;
 
 	/* We need this, since &ap doesn't work on some platforms... */
 	va_copy(cpy, ap);
-	guide = parse_guide(buffer, tok, guide, &cpy);
+	errmsg = parse_guide(buffer, tok, &guide, &cpy);
 	va_end(cpy);
 
-	if (!guide)
-		return false;
+	if (errmsg) {
+		return tal_fmt(ctx, "Parsing '%.*s': %s",
+			       (int)(guide - orig_guide), orig_guide,
+			       errmsg);
+	}
 	assert(guide[0] == '\0');
-	return true;
+	return NULL;
 }
 
-bool json_scan(const char *buffer,
-	       const jsmntok_t *tok,
-	       const char *guide,
-	       ...)
+const char *json_scan(const tal_t *ctx,
+		      const char *buffer,
+		      const jsmntok_t *tok,
+		      const char *guide,
+		      ...)
 {
 	va_list ap;
-	bool ret;
+	const char *ret;
 
 	va_start(ap, guide);
-	ret = json_scanv(buffer, tok, guide, ap);
+	ret = json_scanv(ctx, buffer, tok, guide, ap);
 	va_end(ap);
 	return ret;
 }

common/json.h

@@ -152,15 +152,18 @@ jsmntok_t *json_tok_copy(const tal_t *ctx, const jsmntok_t *tok);
 void json_tok_remove(jsmntok_t **tokens,
 		     jsmntok_t *obj_or_array, const jsmntok_t *tok, size_t num);
 
-/* Guide is % for a token: each must be followed by JSON_SCAN(). */
-bool json_scan(const char *buffer,
-	       const jsmntok_t *tok,
-	       const char *guide,
-	       ...);
+/* Guide is % for a token: each must be followed by JSON_SCAN().
+ * Returns NULL on error (asserts() on bad guide). */
+const char *json_scan(const tal_t *ctx,
+		      const char *buffer,
+		      const jsmntok_t *tok,
+		      const char *guide,
+		      ...);
 
 /* eg. JSON_SCAN(json_to_bool, &boolvar) */
 #define JSON_SCAN(fmt, var)						\
 	json_scan,							\
+	stringify(fmt),							\
 	((var) + 0*sizeof(fmt((const char *)NULL,			\
 			      (const jsmntok_t *)NULL, var) == true)),	\
 	(fmt)
@@ -168,16 +171,18 @@ bool json_scan(const char *buffer,
 /* eg. JSON_SCAN_TAL(tmpctx, json_strdup, &charvar) */
 #define JSON_SCAN_TAL(ctx, fmt, var)					\
 	(ctx),								\
+	stringify(fmt),							\
 	((var) + 0*sizeof((*var) = fmt((ctx),				\
 				       (const char *)NULL,		\
 				       (const jsmntok_t *)NULL))),	\
 	(fmt)
 
 /* Already-have-varargs version */
-bool json_scanv(const char *buffer,
-		const jsmntok_t *tok,
-		const char *guide,
-		va_list ap);
+const char *json_scanv(const tal_t *ctx,
+		       const char *buffer,
+		       const jsmntok_t *tok,
+		       const char *guide,
+		       va_list ap);
 
 /* Iterator macro for array: i is counter, t is token ptr, arr is JSMN_ARRAY */
 #define json_for_each_arr(i, t, arr) \

common/test/run-json_scan.c

@@ -137,6 +137,7 @@ int main(int argc, char *argv[])
 	char *s;
 	u32 u32val;
 	u8 *hex;
+	const char *err;
 
 	common_setup(argv[0]);
 
@@ -146,55 +147,70 @@ int main(int argc, char *argv[])
 	assert(toks->size == 4);
 
 	/* These are direct matches, and they work. */
-	assert(json_scan(buf, toks, "{1:one}"));
-	assert(json_scan(buf, toks, "{1:one,2:two}"));
-	assert(json_scan(buf, toks, "{2:two,1:one}"));
-	assert(json_scan(buf, toks, "{2:two,1:one,3:{three:{deeper:17}}}"));
-	assert(json_scan(buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[0:{1:arrone}]}"));
-	assert(json_scan(buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2]}"));
-	assert(json_scan(buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2,2:[0:3,1:4]]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{1:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{1:one,2:two}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}}}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[0:{1:arrone}]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2]}"));
+	assert(!json_scan(tmpctx, buf, toks, "{2:two,1:one,3:{three:{deeper:17}},arr:[1:2,2:[0:3,1:4]]}"));
 
 	/* These do not match */
-	assert(!json_scan(buf, toks, "{2:one}"));
-	assert(!json_scan(buf, toks, "{1:one,2:tw}"));
-	assert(!json_scan(buf, toks, "{1:one,2:twoo}"));
-	assert(!json_scan(buf, toks, "{4:one}"));
-	assert(!json_scan(buf, toks, "{2:two,1:one,3:three}"));
-	assert(!json_scan(buf, toks, "{3:{three:deeper}}"));
-	assert(!json_scan(buf, toks, "{3:{three:{deeper:{}}}}"));
-	assert(!json_scan(buf, toks, "{arr:{}}"));
-	assert(!json_scan(buf, toks, "{arr:[0:1]}"));
-	assert(!json_scan(buf, toks, "{arr:[4:0]}"));
-	assert(!json_scan(buf, toks, "{arr:[0:{1:arrtwo}]}"));
-	assert(!json_scan(buf, toks, "{arr:[1:3]}"));
-	assert(!json_scan(buf, toks, "{arr:[2:[0:1]]}"));
+	err = json_scan(tmpctx, buf, toks, "{2:one}");
+	assert(streq(err, "Parsing '{2:one': \"two\" does not match expected one"));
+	err = json_scan(tmpctx, buf, toks, "{1:one,2:tw}");
+	assert(streq(err, "Parsing '{1:one,2:tw': \"two\" does not match expected tw"));
+	err = json_scan(tmpctx, buf, toks, "{1:one,2:twoo}");
+	assert(streq(err, "Parsing '{1:one,2:twoo': \"two\" does not match expected twoo"));
+	err = json_scan(tmpctx, buf, toks, "{4:one}");
+	assert(streq(err, "Parsing '{4:': object does not have member 4"));
+	err = json_scan(tmpctx, buf, toks, "{2:two,1:one,3:three}");
+	assert(streq(err, "Parsing '{2:two,1:one,3:three': {\"three\": {\"deeper\": 17}} does not match expected three"));
+	err = json_scan(tmpctx, buf, toks, "{3:{three:deeper}}");
+	assert(streq(err, "Parsing '{3:{three:deeper': {\"deeper\": 17} does not match expected deeper"));
+	err = json_scan(tmpctx, buf, toks, "{3:{three:{deeper:{}}}}");
+	assert(streq(err, "Parsing '{3:{three:{deeper:{': token is not an object: 17"));
+	err = json_scan(tmpctx, buf, toks, "{arr:{}}");
+	assert(streq(err, "Parsing '{arr:{': token is not an object: [{\"1\": \"arrone\"}, 2, [3, 4]]"));
+	err = json_scan(tmpctx, buf, toks, "{arr:[0:1]}");
+	assert(streq(err, "Parsing '{arr:[0:1': {\"1\": \"arrone\"} does not match expected 1"));
+	err = json_scan(tmpctx, buf, toks, "{arr:[4:0]}");
+	assert(streq(err, "Parsing '{arr:[4:': token has no index 4: [{\"1\": \"arrone\"}, 2, [3, 4]]"));
+	err = json_scan(tmpctx, buf, toks, "{arr:[0:{1:arrtwo}]}");
+	assert(streq(err, "Parsing '{arr:[0:{1:arrtwo': \"arrone\" does not match expected arrtwo"));
+	err = json_scan(tmpctx, buf, toks, "{arr:[1:3]}");
+	assert(streq(err, "Parsing '{arr:[1:3': 2 does not match expected 3"));
+	err = json_scan(tmpctx, buf, toks, "{arr:[2:[0:1]]}");
+	assert(streq(err, "Parsing '{arr:[2:[0:1': 3 does not match expected 1"));
 
 	/* These capture simple values. */
-	assert(json_scan(buf, toks, "{3:{three:{deeper:%}}}",
-			 JSON_SCAN(json_to_number, &u32val)));
+	assert(!json_scan(tmpctx, buf, toks, "{3:{three:{deeper:%}}}",
+			  JSON_SCAN(json_to_number, &u32val)));
 	assert(u32val == 17);
-	assert(!json_scan(buf, toks, "{1:%}",
-			 JSON_SCAN(json_to_number, &u32val)));
-	assert(json_scan(buf, toks, "{1:%}",
-			 JSON_SCAN_TAL(tmpctx, json_strdup, &s)));
+	err = json_scan(tmpctx, buf, toks, "{1:%}",
+			JSON_SCAN(json_to_number, &u32val));
+	assert(streq(err, "Parsing '{1:%': json_to_number could not parse \"one\""));
+	assert(!json_scan(tmpctx, buf, toks, "{1:%}",
+			  JSON_SCAN_TAL(tmpctx, json_strdup, &s)));
 	assert(tal_parent(s) == tmpctx);
 	assert(streq(s, "one"));
 
-	assert(!json_scan(buf, toks, "{1:%}",
-			  JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex, &hex)));
+	err = json_scan(tmpctx, buf, toks, "{1:%}",
+			JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex, &hex));
+	assert(streq(err, "Parsing '{1:%': json_tok_bin_from_hex could not parse \"one\""));
 
-	assert(json_scan(buf, toks, "{3:%}", JSON_SCAN(json_to_tok, &t)));
+	assert(!json_scan(tmpctx, buf, toks, "{3:%}", JSON_SCAN(json_to_tok, &t)));
 	assert(t == &toks[6]);
 
-	assert(json_scan(buf, toks, "{arr:%}", JSON_SCAN(json_to_tok, &t)));
+	assert(!json_scan(tmpctx, buf, toks, "{arr:%}", JSON_SCAN(json_to_tok, &t)));
 	assert(t == &toks[12]);
 
-	assert(json_scan(buf, toks, "{arr:[1:%]}",
-			 JSON_SCAN(json_to_number, &u32val)));
+	assert(!json_scan(tmpctx, buf, toks, "{arr:[1:%]}",
+			  JSON_SCAN(json_to_number, &u32val)));
 	assert(u32val == 2);
 
-	assert(json_scan(buf, toks, "{arr:[2:[1:%]]}",
-			 JSON_SCAN(json_to_number, &u32val)));
+	assert(!json_scan(tmpctx, buf, toks, "{arr:[2:[1:%]]}",
+			  JSON_SCAN(json_to_number, &u32val)));
 	assert(u32val == 4);
 
 	common_shutdown();

lightningd/bitcoind.c

@@ -265,20 +265,24 @@ static void sendrawtx_callback(const char *buf, const jsmntok_t *toks,
 			       const jsmntok_t *idtok,
 			       struct sendrawtx_call *call)
 {
+	const char *err;
 	const char *errmsg = NULL;
 	bool success = false;
 
-	if (!json_scan(buf, toks, "{result:{success:%}}",
-		       JSON_SCAN(json_to_bool, &success))) {
+	err = json_scan(tmpctx, buf, toks, "{result:{success:%}}",
+			JSON_SCAN(json_to_bool, &success));
+	if (err) {
 		bitcoin_plugin_error(call->bitcoind, buf, toks,
 				     "sendrawtransaction",
-				     "bad 'result' field");
+				     "bad 'result' field: %s", err);
 	} else if (!success) {
-		if (!json_scan(buf, toks, "{result:{errmsg:%}}",
-			       JSON_SCAN_TAL(tmpctx, json_strdup, &errmsg)))
+		err = json_scan(tmpctx, buf, toks, "{result:{errmsg:%}}",
+				JSON_SCAN_TAL(tmpctx, json_strdup, &errmsg));
+		if (err)
 			bitcoin_plugin_error(call->bitcoind, buf, toks,
 					     "sendrawtransaction",
-					     "bad 'errmsg' field");
+					     "bad 'errmsg' field: %s",
+					     err);
 	}
 
 	db_begin_transaction(call->bitcoind->ld->wallet->db);
@@ -311,8 +315,8 @@ static void sendrawtx_compatv090_callback(const char *buf,
 	 * because it is an old plugin which does not support the
 	 * new `allowhighfees` parameter.
 	 */
-	if (!json_scan(buf, toks, "{error:{code:"
-		       stringify(JSONRPC2_INVALID_PARAMS)"}}")) {
+	if (json_scan(tmpctx, buf, toks, "{error:{code:"
+		      stringify(JSONRPC2_INVALID_PARAMS)"}}") != NULL) {
 		goto fallback;
 	}
 
@@ -407,25 +411,27 @@ getrawblockbyheight_callback(const char *buf, const jsmntok_t *toks,
 			     const jsmntok_t *idtok,
 			     struct getrawblockbyheight_call *call)
 {
-	const char *block_str;
+	const char *block_str, *err;
 	struct bitcoin_blkid blkid;
 	struct bitcoin_block *blk;
 
 	/* If block hash is `null`, this means not found! Call the callback
 	 * with NULL values. */
-	if (json_scan(buf, toks, "{result:{blockhash:null}}")) {
+	err = json_scan(tmpctx, buf, toks, "{result:{blockhash:null}}");
+	if (!err) {
 		db_begin_transaction(call->bitcoind->ld->wallet->db);
 		call->cb(call->bitcoind, NULL, NULL, call->cb_arg);
 		db_commit_transaction(call->bitcoind->ld->wallet->db);
 		goto clean;
 	}
 
-	if (!json_scan(buf, toks, "{result:{blockhash:%,block:%}}",
-		       JSON_SCAN(json_to_sha256, &blkid.shad.sha),
-		       JSON_SCAN_TAL(tmpctx, json_strdup, &block_str)))
+	err = json_scan(tmpctx, buf, toks, "{result:{blockhash:%,block:%}}",
+			JSON_SCAN(json_to_sha256, &blkid.shad.sha),
+			JSON_SCAN_TAL(tmpctx, json_strdup, &block_str));
+	if (err)
 		bitcoin_plugin_error(call->bitcoind, buf, toks,
 				     "getrawblockbyheight",
-				     "bad 'result' field");
+				     "bad 'result' field: %s", err);
 
 	blk = bitcoin_block_from_hex(tmpctx, chainparams, block_str,
 				     strlen(block_str));
@@ -499,18 +505,19 @@ static void getchaininfo_callback(const char *buf, const jsmntok_t *toks,
 				  const jsmntok_t *idtok,
 				  struct getchaininfo_call *call)
 {
-	const char *chain;
+	const char *err, *chain;
 	u32 headers, blocks;
 	bool ibd;
 
-	if (!json_scan(buf, toks,
-		       "{result:{chain:%,headercount:%,blockcount:%,ibd:%}}",
-		       JSON_SCAN_TAL(tmpctx, json_strdup, &chain),
-		       JSON_SCAN(json_to_number, &headers),
-		       JSON_SCAN(json_to_number, &blocks),
-		       JSON_SCAN(json_to_bool, &ibd)))
+	err = json_scan(tmpctx, buf, toks,
+			"{result:{chain:%,headercount:%,blockcount:%,ibd:%}}",
+			JSON_SCAN_TAL(tmpctx, json_strdup, &chain),
+			JSON_SCAN(json_to_number, &headers),
+			JSON_SCAN(json_to_number, &blocks),
+			JSON_SCAN(json_to_bool, &ibd));
+	if (err)
 		bitcoin_plugin_error(call->bitcoind, buf, toks, "getchaininfo",
-				     "bad 'result' field");
+				     "bad 'result' field: %s", err);
 
 	db_begin_transaction(call->bitcoind->ld->wallet->db);
 	call->cb(call->bitcoind, chain, headers, blocks, ibd,
@@ -570,21 +577,24 @@ static void getutxout_callback(const char *buf, const jsmntok_t *toks,
 			      const jsmntok_t *idtok,
 			      struct getutxout_call *call)
 {
+	const char *err;
 	struct bitcoin_tx_output txout;
 
-	if (json_scan(buf, toks, "{result:{script:null}}")) {
+	err = json_scan(tmpctx, buf, toks, "{result:{script:null}}");
+	if (!err) {
 		db_begin_transaction(call->bitcoind->ld->wallet->db);
 		call->cb(call->bitcoind, NULL, call->cb_arg);
 		db_commit_transaction(call->bitcoind->ld->wallet->db);
 		goto clean;
 	}
 
-	if (!json_scan(buf, toks, "{result:{script:%,amount:%}}",
-		       JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex,
-				     &txout.script),
-		       JSON_SCAN(json_to_sat, &txout.amount)))
+	err = json_scan(tmpctx, buf, toks, "{result:{script:%,amount:%}}",
+			JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex,
+				      &txout.script),
+			JSON_SCAN(json_to_sat, &txout.amount));
+	if (err)
 		bitcoin_plugin_error(call->bitcoind, buf, toks, "getutxout",
-				     "bad 'result' field");
+				     "bad 'result' field: %s", err);
 
 	db_begin_transaction(call->bitcoind->ld->wallet->db);
 	call->cb(call->bitcoind, &txout, call->cb_arg);

plugins/bcli.c

@@ -326,6 +326,7 @@ static struct command_result *process_getutxout(struct bitcoin_cli *bcli)
 	const jsmntok_t *tokens;
 	struct json_stream *response;
 	struct bitcoin_tx_output output;
+	const char *err;
 
 	/* As of at least v0.15.1.0, bitcoind returns "success" but an empty
 	   string on a spent txout. */
@@ -343,14 +344,14 @@ static struct command_result *process_getutxout(struct bitcoin_cli *bcli)
 		return command_err_bcli_badjson(bcli, "cannot parse");
 	}
 
-	if (!json_scan(bcli->output, tokens,
+	err = json_scan(tmpctx, bcli->output, tokens,
 		       "{value:%,scriptPubKey:{hex:%}}",
 		       JSON_SCAN(json_to_bitcoin_amount,
 				 &output.amount.satoshis), /* Raw: bitcoind */
 		       JSON_SCAN_TAL(bcli, json_tok_bin_from_hex,
-				     &output.script))) {
-		return command_err_bcli_badjson(bcli, "cannot scan");
-	}
+				     &output.script));
+	if (err)
+		return command_err_bcli_badjson(bcli, err);
 
 	response = jsonrpc_stream_success(bcli->cmd);
 	json_add_amount_sat_only(response, "amount", output.amount);
@@ -365,7 +366,7 @@ static struct command_result *process_getblockchaininfo(struct bitcoin_cli *bcli
 	struct json_stream *response;
 	bool ibd;
 	u32 headers, blocks;
-	const char *chain;
+	const char *chain, *err;
 
 	tokens = json_parse_simple(bcli->output,
 				   bcli->output, bcli->output_bytes);
@@ -373,14 +374,14 @@ static struct command_result *process_getblockchaininfo(struct bitcoin_cli *bcli
 		return command_err_bcli_badjson(bcli, "cannot parse");
 	}
 
-	if (!json_scan(bcli->output, tokens,
-		       "{chain:%,headers:%,blocks:%,initialblockdownload:%}",
-		       JSON_SCAN_TAL(tmpctx, json_strdup, &chain),
-		       JSON_SCAN(json_to_number, &headers),
-		       JSON_SCAN(json_to_number, &blocks),
-		       JSON_SCAN(json_to_bool, &ibd))) {
-		return command_err_bcli_badjson(bcli, "cannot scan");
-	}
+	err = json_scan(tmpctx, bcli->output, tokens,
+			"{chain:%,headers:%,blocks:%,initialblockdownload:%}",
+			JSON_SCAN_TAL(tmpctx, json_strdup, &chain),
+			JSON_SCAN(json_to_number, &headers),
+			JSON_SCAN(json_to_number, &blocks),
+			JSON_SCAN(json_to_bool, &ibd));
+	if (err)
+		return command_err_bcli_badjson(bcli, err);
 
 	response = jsonrpc_stream_success(bcli->cmd);
 	json_add_string(response, "chain", chain);
@@ -425,8 +426,8 @@ estimatefees_parse_feerate(struct bitcoin_cli *bcli, u64 *feerate)
 		return command_err_bcli_badjson(bcli, "cannot parse");
 	}
 
-	if (!json_scan(bcli->output, tokens, "{feerate:%}",
-		       JSON_SCAN(json_to_bitcoin_amount, feerate))) {
+	if (json_scan(tmpctx, bcli->output, tokens, "{feerate:%}",
+		      JSON_SCAN(json_to_bitcoin_amount, feerate)) != NULL) {
 		/* Paranoia: if it had a feerate, but was malformed: */
 		if (json_get_member(bcli->output, tokens, "feerate"))
 			return command_err_bcli_badjson(bcli, "cannot scan");
@@ -782,6 +783,7 @@ static void parse_getnetworkinfo_result(struct plugin *p, const char *buf)
 	const jsmntok_t *result;
 	bool tx_relay;
 	u32 min_version = 160000;
+	const char *err;
 
 	result = json_parse_simple(NULL, buf, strlen(buf));
 	if (!result)
@@ -790,12 +792,14 @@ static void parse_getnetworkinfo_result(struct plugin *p, const char *buf)
 			      gather_args(bitcoind, "getnetworkinfo", NULL), buf);
 
 	/* Check that we have a fully-featured `estimatesmartfee`. */
-	if (!json_scan(buf, result, "{version:%,localrelay:%}",
-		       JSON_SCAN(json_to_u32, &bitcoind->version),
-		       JSON_SCAN(json_to_bool, &tx_relay)))
-		plugin_err(p, "No 'version' or localrelay in '%s' ? Got '%s'. Can not"
-			      " continue without proceeding to sanity checks.",
-			      gather_args(bitcoind, "getnetworkinfo", NULL), buf);
+	err = json_scan(tmpctx, buf, result, "{version:%,localrelay:%}",
+			JSON_SCAN(json_to_u32, &bitcoind->version),
+			JSON_SCAN(json_to_bool, &tx_relay));
+	if (err)
+		plugin_err(p, "%s.  Got '%s'. Can not"
+			   " continue without proceeding to sanity checks.",
+			   err,
+			   gather_args(bitcoind, "getnetworkinfo", NULL), buf);
 
 	if (bitcoind->version < min_version)
 		plugin_err(p, "Unsupported bitcoind version %"PRIu32", at least"

plugins/keysend.c

@@ -251,10 +251,13 @@ static struct command_result *htlc_accepted_call(struct command *cmd,
 	struct keysend_in *ki;
 	struct out_req *req;
 	struct timeabs now = time_now();
+	const char *err;
 
-	if (!json_scan(buf, params, "{onion:{payload:%},htlc:{payment_hash:%}}",
-		       JSON_SCAN_TAL(cmd, json_tok_bin_from_hex, &rawpayload),
-		       JSON_SCAN(json_to_sha256, &payment_hash)))
+	err = json_scan(tmpctx, buf, params,
+			"{onion:{payload:%},htlc:{payment_hash:%}}",
+			JSON_SCAN_TAL(cmd, json_tok_bin_from_hex, &rawpayload),
+			JSON_SCAN(json_to_sha256, &payment_hash));
+	if (err)
 		return htlc_accepted_continue(cmd, NULL);
 
 	max = tal_bytelen(rawpayload);

plugins/libplugin.c

@@ -498,6 +498,7 @@ void rpc_scan(struct plugin *plugin,
 	      ...)
 {
 	bool error;
+	const char *err;
 	const jsmntok_t *contents;
 	int reqlen;
 	const char *p;
@@ -515,12 +516,13 @@ void rpc_scan(struct plugin *plugin,
 	p = membuf_consume(&plugin->rpc_conn->mb, reqlen);
 
 	va_start(ap, guide);
-	error = !json_scanv(p, contents, guide, ap);
+	err = json_scanv(tmpctx, p, contents, guide, ap);
 	va_end(ap);
 
-	if (error)
-		plugin_err(plugin, "Could not parse %s in reply to %s: '%.*s'",
-			   guide, method, reqlen, membuf_elems(&plugin->rpc_conn->mb));
+	if (err)
+		plugin_err(plugin, "Could not parse %s in reply to %s: %s: '%.*s'",
+			   guide, method, err,
+			   reqlen, membuf_elems(&plugin->rpc_conn->mb));
 }
 
 static void handle_rpc_reply(struct plugin *plugin, const jsmntok_t *toks)
@@ -810,19 +812,21 @@ static struct command_result *handle_init(struct command *cmd,
 	char *dir, *network;
 	struct plugin *p = cmd->plugin;
 	bool with_rpc = p->rpc_conn != NULL;
+	const char *err;
 
 	configtok = json_get_member(buf, params, "configuration");
-	if (!json_scan(buf, configtok,
-		       "{lightning-dir:%"
-		       ",network:%"
-		       ",feature_set:%"
-		       ",rpc-file:%}",
-		       JSON_SCAN_TAL(tmpctx, json_strdup, &dir),
-		       JSON_SCAN_TAL(tmpctx, json_strdup, &network),
-		       JSON_SCAN_TAL(p, json_to_feature_set, &p->our_features),
-		       JSON_SCAN_TAL(p, json_strdup, &p->rpc_location)))
-		plugin_err(p, "cannot scan init params: %.*s",
-			   json_tok_full_len(params),
+	err = json_scan(tmpctx, buf, configtok,
+			"{lightning-dir:%"
+			",network:%"
+			",feature_set:%"
+			",rpc-file:%}",
+			JSON_SCAN_TAL(tmpctx, json_strdup, &dir),
+			JSON_SCAN_TAL(tmpctx, json_strdup, &network),
+			JSON_SCAN_TAL(p, json_to_feature_set, &p->our_features),
+			JSON_SCAN_TAL(p, json_strdup, &p->rpc_location));
+	if (err)
+		plugin_err(p, "cannot scan init params: %s: %.*s",
+			   err, json_tok_full_len(params),
 			   json_tok_full(buf, params));
 
 	/* Move into lightning directory: other files are relative */

plugins/pay.c

@@ -399,9 +399,9 @@ get_remote_block_height(const char *buf, const jsmntok_t *error)
 	u16 type;
 
 	/* Is there even a raw_message?  */
-	if (!json_scan(buf, error, "{data:{raw_message:%}}",
-		       JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex,
-				     &raw_message)))
+	if (json_scan(tmpctx, buf, error, "{data:{raw_message:%}}",
+		      JSON_SCAN_TAL(tmpctx, json_tok_bin_from_hex,
+				    &raw_message)) != NULL)
 		return 0;
 
 	/* BOLT #4:
@@ -431,19 +431,24 @@ static struct command_result *waitsendpay_error(struct command *cmd,
 	struct pay_attempt *attempt = current_attempt(pc);
 	errcode_t code;
 	int failcode;
+	const char *err;
 
 	attempt_failed_tok(pc, "waitsendpay", buf, error);
 
-	if (!json_scan(buf, error, "{code:%}",
-		       JSON_SCAN(json_to_errcode, &code)))
-		plugin_err(cmd->plugin, "waitsendpay error gave no 'code'? '%.*s'",
-			   error->end - error->start, buf + error->start);
+	err = json_scan(tmpctx, buf, error, "{code:%}",
+			JSON_SCAN(json_to_errcode, &code));
+	if (err)
+		plugin_err(cmd->plugin, "waitsendpay error %s? '%.*s'",
+			   err,
+			   json_tok_full_len(error), json_tok_full(buf, error));
 
 	if (code != PAY_UNPARSEABLE_ONION) {
-		if (!json_scan(buf, error, "{data:{failcode:%}}",
-			       JSON_SCAN(json_to_int, &failcode)))
-			plugin_err(cmd->plugin, "waitsendpay error gave no 'failcode'? '%.*s'",
-				   error->end - error->start, buf + error->start);
+		err = json_scan(tmpctx, buf, error, "{data:{failcode:%}}",
+				JSON_SCAN(json_to_int, &failcode));
+		if (err)
+			plugin_err(cmd->plugin, "waitsendpay failcode error %s '%.*s'",
+				   err,
+				   json_tok_full_len(error), json_tok_full(buf, error));
 	}
 
 	/* Special case for WIRE_INCORRECT_OR_UNKNOWN_PAYMENT_DETAILS.
@@ -510,12 +515,15 @@ static struct command_result *waitsendpay_error(struct command *cmd,
 
 	if (failcode & NODE) {
 		struct node_id id;
-		const char *idstr;
+		const char *idstr, *err;
 
-		if (!json_scan(buf, error, "{data:{erring_node:%}}",
-			       JSON_SCAN(json_to_node_id, &id)))
-			plugin_err(cmd->plugin, "waitsendpay error no erring_node '%.*s'",
-				   error->end - error->start, buf + error->start);
+		err = json_scan(tmpctx, buf, error, "{data:{erring_node:%}}",
+				JSON_SCAN(json_to_node_id, &id));
+		if (err)
+			plugin_err(cmd->plugin, "waitsendpay error %s '%.*s'",
+				   err,
+				   json_tok_full_len(error),
+				   json_tok_full(buf, error));
 
 		/* FIXME: Keep as node_id, don't use strings. */
 		idstr = node_id_to_hexstr(tmpctx, &id);
@@ -530,13 +538,17 @@ static struct command_result *waitsendpay_error(struct command *cmd,
 	} else {
 		struct short_channel_id scid;
 		u32 dir;
-		const char *scidstr;
-
-		if (!json_scan(buf, error, "{data:{erring_channel:%,erring_direction:%}}",
-			       JSON_SCAN(json_to_short_channel_id, &scid),
-			       JSON_SCAN(json_to_number, &dir)))
-			plugin_err(cmd->plugin, "waitsendpay error no erring_channel/direction '%.*s'",
-				   error->end - error->start, buf + error->start);
+		const char *scidstr, *err;
+
+		err = json_scan(tmpctx, buf, error,
+				"{data:{erring_channel:%,erring_direction:%}}",
+				JSON_SCAN(json_to_short_channel_id, &scid),
+				JSON_SCAN(json_to_number, &dir));
+		if (err)
+			plugin_err(cmd->plugin, "waitsendpay error %s '%.*s'",
+				   err,
+				   json_tok_full_len(error),
+				   json_tok_full(buf, error));
 
 		scidstr = short_channel_id_to_str(tmpctx, &scid);
 		/* If failure is in routehint part, try next one */
@@ -736,6 +748,7 @@ static struct command_result *getroute_done(struct command *cmd,
 	struct amount_msat max_fee;
 	u32 delay;
 	struct out_req *req;
+	const char *err;
 
 	if (!t)
 		plugin_err(cmd->plugin, "getroute gave no 'route'? '%.*s'",
@@ -754,11 +767,13 @@ static struct command_result *getroute_done(struct command *cmd,
 	} else
 		attempt->route = json_strdup(pc->ps->attempts, buf, t);
 
-	if (!json_scan(buf, t, "[0:{msatoshi:%,delay:%}]",
-		       JSON_SCAN(json_to_msat, &fee),
-		       JSON_SCAN(json_to_number, &delay)))
-		plugin_err(cmd->plugin, "getroute with invalid msatoshi/delay? %.*s",
-			   result->end - result->start, buf);
+	err = json_scan(tmpctx, buf, t, "[0:{msatoshi:%,delay:%}]",
+			JSON_SCAN(json_to_msat, &fee),
+			JSON_SCAN(json_to_number, &delay));
+	if (err)
+		plugin_err(cmd->plugin, "getroute %s? %.*s",
+			   err,
+			   json_tok_full_len(result), json_tok_full(buf, result));
 
 	if (pc->maxfee_pct_millionths / 100 > UINT32_MAX)
 		plugin_err(cmd->plugin, "max fee percent too large: %lf",

plugins/spender/openchannel.c

@@ -547,15 +547,17 @@ static void json_peer_sigs(struct command *cmd,
 	struct channel_id cid;
 	const struct wally_psbt *psbt;
 	struct multifundchannel_destination *dest;
-
-	if (!json_scan(buf, params,
-		       "{openchannel_peer_sigs:"
-		       "{channel_id:%,signed_psbt:%}}",
-		       JSON_SCAN(json_to_channel_id, &cid),
-		       JSON_SCAN_TAL(cmd, json_to_psbt, &psbt)))
+	const char *err;
+
+	err = json_scan(tmpctx, buf, params,
+			"{openchannel_peer_sigs:"
+			"{channel_id:%,signed_psbt:%}}",
+			JSON_SCAN(json_to_channel_id, &cid),
+			JSON_SCAN_TAL(cmd, json_to_psbt, &psbt));
+	if (err)
 		plugin_err(cmd->plugin,
-			   "`openchannel_peer_sigs` did not scan",
-			   json_tok_full_len(params),
+			   "`openchannel_peer_sigs` did not scan: %s",
+			   err, json_tok_full_len(params),
 			   json_tok_full(buf, params));
 
 	/* Find the destination that's got this channel_id on it! */