common: generalize json_tok_remove.

It assumes the head of the array is the object/array we want to remove from,
but that's not true if we're trying to remove from a sub-object.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

common/json.c

@@ -292,24 +292,37 @@ jsmntok_t *json_tok_copy(const tal_t *ctx, const jsmntok_t *tok)
 	return tal_dup_arr(ctx, jsmntok_t, tok, json_next(tok) - tok, 0);
 }
 
-void json_tok_remove(jsmntok_t **tokens, jsmntok_t *tok, size_t num)
+void json_tok_remove(jsmntok_t **tokens,
+		     jsmntok_t *obj_or_array, const jsmntok_t *tok, size_t num)
 {
-	assert(*tokens);
-	assert((*tokens)->type == JSMN_ARRAY || (*tokens)->type == JSMN_OBJECT);
 	const jsmntok_t *src = tok;
 	const jsmntok_t *end = json_next(*tokens);
-	jsmntok_t *dest = tok;
+	jsmntok_t *dest = *tokens + (tok - *tokens);
 	int remove_count;
 
+	assert(*tokens);
+	assert(obj_or_array->type == JSMN_ARRAY
+	       || obj_or_array->type == JSMN_OBJECT);
+	/* obj_or_array must be inside tokens, and tok must be inside
+	 * obj_or_array */
+	assert(obj_or_array >= *tokens
+	       && obj_or_array < *tokens + tal_count(*tokens));
+	assert(tok >= obj_or_array
+	       && tok < *tokens + tal_count(*tokens));
+
 	for (int i = 0; i < num; i++)
 		src = json_next(src);
 
+	/* Don't give us a num which goes over end of obj_or_array. */
+	assert(src <= json_next(obj_or_array));
+
 	remove_count = src - tok;
 
 	memmove(dest, src, sizeof(jsmntok_t) * (end - src));
 
+	/* Subtract first: this ptr may move after tal_resize! */
+	obj_or_array->size -= num;
 	tal_resize(tokens, tal_count(*tokens) - remove_count);
-	(*tokens)->size -= num;
 }
 
 const jsmntok_t *json_delve(const char *buffer,

common/json.h

@@ -75,10 +75,11 @@ void json_tok_print(const char *buffer, const jsmntok_t *params);
 jsmntok_t *json_tok_copy(const tal_t *ctx, const jsmntok_t *tok);
 
 /*
- * Remove @num json values from a json array or object. @tok points
- * to the first value to remove.  The array will be resized.
+ * Remove @num json values from a json array or object @obj. @tok points
+ * to the first value to remove.  The array @tokens will be resized.
  */
-void json_tok_remove(jsmntok_t **tokens, jsmntok_t *tok, size_t num);
+void json_tok_remove(jsmntok_t **tokens,
+		     jsmntok_t *obj_or_array, const jsmntok_t *tok, size_t num);
 
 /* Guide is a string with . for members, [] around indexes. */
 const jsmntok_t *json_delve(const char *buffer,

common/test/run-json_remove.c

@@ -42,6 +42,11 @@ static void test_toks(const struct json *j, ...)
 	va_list(ap);
 	va_start(ap, j);
 	while ((value = va_arg(ap, char *)) != NULL) {
+		if (tok->type == JSMN_OBJECT)
+			assert(streq(value, "{"));
+		else if (tok->type == JSMN_ARRAY)
+			assert(streq(value, "["));
+		else
 			assert(json_tok_streq(j->buffer, tok, value));
 		tok++;
 	}
@@ -50,14 +55,14 @@ static void test_toks(const struct json *j, ...)
 static void sanity(void)
 {
 	struct json *j = json_parse(tmpctx, "[]");
-	json_tok_remove(&j->toks, j->toks, 0);
+	json_tok_remove(&j->toks, j->toks, j->toks, 0);
 	assert(j);
 }
 
 static void remove_one(void)
 {
 	struct json *j = json_parse(tmpctx, "['invoice']");
-	json_tok_remove(&j->toks, j->toks + 1, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 1);
 	assert(j);
 }
 
@@ -65,7 +70,7 @@ static void remove_first(void)
 {
 	struct json *j = json_parse(tmpctx, "['one', 'two', 'three']");
 	assert(j);
-	json_tok_remove(&j->toks, j->toks + 1, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 1);
 	assert(j->toks);
 
 	test_toks(j, "two", "three", NULL);
@@ -74,13 +79,13 @@ static void remove_first(void)
 	j = json_parse(tmpctx, "{'1':'one', '2':'two', '3':'three'}");
 	assert(j);
 
-	json_tok_remove(&j->toks, j->toks + 1, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 1);
 	assert(j);
 	test_toks(j, "2", "two", "3", "three", NULL);
 	assert(tal_count(j->toks) == 5);
 
 	j = json_parse(tmpctx, "{'1':'one', '2':'two', '3':'three'}");
-	json_tok_remove(&j->toks, j->toks + 1, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 1);
 	assert(j);
 	test_toks(j, "2", "two", "3", "three",  NULL);
 	assert(tal_count(j->toks) == 5);
@@ -90,12 +95,12 @@ static void remove_first(void)
 static void remove_last(void)
 {
 	struct json *j = json_parse(tmpctx, "['one', 'two', 'three']");
-	json_tok_remove(&j->toks, j->toks + 3, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 3, 1);
 	test_toks(j, "one", "two", NULL);
 	assert(tal_count(j->toks) == 3);
 
 	j = json_parse(tmpctx, "{'1':'one', '2':'two', '3':'three'}");
-	json_tok_remove(&j->toks, j->toks + 5, 1);
+	json_tok_remove(&j->toks, j->toks, j->toks + 5, 1);
 	assert(j);
 	test_toks(j, "1", "one", "2", "two", NULL);
 	assert(tal_count(j->toks) == 5);
@@ -104,15 +109,15 @@ static void remove_last(void)
 static void remove_multiple(void)
 {
 	struct json *j = json_parse(tmpctx, "['a', 'b', 'c', 'd', 'e']");
-	json_tok_remove(&j->toks, j->toks + 1, 2);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 2);
 	test_toks(j, "c", "d", "e", NULL);
 
 	j = json_parse(tmpctx, "['a', 'b', 'c', 'd', 'e']");
-	json_tok_remove(&j->toks, j->toks + 2, 2);
+	json_tok_remove(&j->toks, j->toks, j->toks + 2, 2);
 	test_toks(j, "a", "d", "e", NULL);
 
 	j = json_parse(tmpctx, "{'1':'one', '2':'two', '3':'three', '4':'four'}");
-	json_tok_remove(&j->toks, j->toks + 3, 2);
+	json_tok_remove(&j->toks, j->toks, j->toks + 3, 2);
 	assert(j);
 	test_toks(j, "1", "one", "4", "four", NULL);
 	assert(tal_count(j->toks) == 5);
@@ -121,11 +126,11 @@ static void remove_multiple(void)
 static void remove_all(void)
 {
 	struct json *j = json_parse(tmpctx, "['a', 'b', 'c', 'd', 'e']");
-	json_tok_remove(&j->toks, j->toks + 1, 5);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 5);
 	assert(tal_count(j->toks) == 1);
 
 	j = json_parse(tmpctx, "{'1':'one', '2':'two', '3':'three', '4':'four'}");
-	json_tok_remove(&j->toks, j->toks + 1, 4);
+	json_tok_remove(&j->toks, j->toks, j->toks + 1, 4);
 	assert(tal_count(j->toks) == 1);
 }
 
@@ -137,10 +142,24 @@ static void remove_complex(void)
 		"'4': { '4.1': 'a', '4.2':'b', '4.3':'c' }, "
 	"'5':'five'}");
 
-	json_tok_remove(&j->toks, j->toks + 5, 2);
+	json_tok_remove(&j->toks, j->toks, j->toks + 5, 2);
 	test_toks(j, "1", "one", "2", "two", "5", "five", NULL);
 }
 
+static void remove_inside_obj(void)
+{
+	jsmntok_t *tok;
+	struct json *j = json_parse(tmpctx,
+	"{'1':'one', '2':'two',"
+		"'3': { '3.1': 'a', '3.2':'b', '3.3':'c' }, "
+	"'4':'four'}");
+
+	tok = (jsmntok_t *)json_get_member(j->buffer, j->toks, "3");
+	json_tok_remove(&j->toks, tok, tok+1, 1);
+	test_toks(j, "1", "one", "2", "two", "3", "{", "3.2", "b", "3.3", "c",
+		  "4", "four", NULL);
+}
+
 int main(void)
 {
 	setup_locale();
@@ -153,6 +172,7 @@ int main(void)
 	remove_multiple();
 	remove_all();
 	remove_complex();
+	remove_inside_obj();
 
 	tal_free(tmpctx);
 	printf("run-json_remove ok\n");

lightningd/jsonrpc.c

@@ -1193,7 +1193,7 @@ static struct command_result *json_check(struct command *cmd,
 	if (params->type == JSMN_OBJECT)
 		name_tok--;
 
-	json_tok_remove(&mod_params, (jsmntok_t *)name_tok, 1);
+	json_tok_remove(&mod_params, mod_params, name_tok, 1);
 
 	cmd->mode = CMD_CHECK;
 	failed = false;