diff --git a/lightningd/Makefile b/lightningd/Makefile index a3fdfa09d..68b3191b5 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -65,6 +65,7 @@ LIGHTNINGD_SRC := \ lightningd/invoice.c \ lightningd/json.c \ lightningd/json_escaped.c \ + lightningd/json_stream.c \ lightningd/jsonrpc.c \ lightningd/lightningd.c \ lightningd/log.c \ diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index 536ac0716..e9db5ce9f 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/lightningd/json.c b/lightningd/json.c index 9d6c87639..f1076c2a0 100644 --- a/lightningd/json.c +++ b/lightningd/json.c @@ -1,4 +1,3 @@ -#include "json.h" #include #include #include @@ -8,7 +7,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -476,184 +477,39 @@ bool json_tok_tok(struct command *cmd, const char *name, return (*out = tok); } -struct json_stream { -#if DEVELOPER - /* tal_arr of types (JSMN_OBJECT/JSMN_ARRAY) we're enclosed in. */ - jsmntype_t *wrapping; -#endif - /* How far to indent. */ - size_t indent; - - /* True if we haven't yet put an element in current wrapping */ - bool empty; - - /* The command we're attached to */ - struct command *cmd; -}; - -static void result_append(struct json_stream *res, const char *str) -{ - struct json_connection *jcon = res->cmd->jcon; - - /* Don't do anything if they're disconnected. */ - if (!jcon) - return; - - jcon_append(jcon, str); -} - -static void PRINTF_FMT(2,3) -result_append_fmt(struct json_stream *res, const char *fmt, ...) -{ - struct json_connection *jcon = res->cmd->jcon; - va_list ap; - - /* Don't do anything if they're disconnected. */ - if (!jcon) - return; - - va_start(ap, fmt); - jcon_append_vfmt(jcon, fmt, ap); - va_end(ap); -} - -static void check_fieldname(const struct json_stream *result, - const char *fieldname) -{ -#if DEVELOPER - size_t n = tal_count(result->wrapping); - if (n == 0) - /* Can't have a fieldname if not in anything! */ - assert(!fieldname); - else if (result->wrapping[n-1] == JSMN_ARRAY) - /* No fieldnames in arrays. */ - assert(!fieldname); - else - /* Must have fieldnames in objects. */ - assert(fieldname); -#endif -} - -static void result_append_indent(struct json_stream *result) -{ - static const char indent_buf[] = " "; - size_t len; - - for (size_t i = 0; i < result->indent * 2; i += len) { - len = result->indent * 2; - if (len > sizeof(indent_buf)-1) - len = sizeof(indent_buf)-1; - /* Use tail of indent_buf string. */ - result_append(result, indent_buf + sizeof(indent_buf) - 1 - len); - } -} - -static void json_start_member(struct json_stream *result, const char *fieldname) -{ - /* Prepend comma if required. */ - if (!result->empty) - result_append(result, ", \n"); - else - result_append(result, "\n"); - - result_append_indent(result); - - check_fieldname(result, fieldname); - if (fieldname) - result_append_fmt(result, "\"%s\": ", fieldname); - result->empty = false; -} - -static void result_indent(struct json_stream *result, jsmntype_t type) -{ -#if DEVELOPER - *tal_arr_expand(&result->wrapping) = type; -#endif - result->empty = true; - result->indent++; -} - -static void result_unindent(struct json_stream *result, jsmntype_t type) -{ - assert(result->indent); -#if DEVELOPER - assert(tal_count(result->wrapping) == result->indent); - assert(result->wrapping[result->indent-1] == type); - tal_resize(&result->wrapping, result->indent-1); -#endif - result->empty = false; - result->indent--; -} - -void json_array_start(struct json_stream *result, const char *fieldname) -{ - json_start_member(result, fieldname); - result_append(result, "["); - result_indent(result, JSMN_ARRAY); -} - -void json_array_end(struct json_stream *result) -{ - result_append(result, "\n"); - result_unindent(result, JSMN_ARRAY); - result_append_indent(result); - result_append(result, "]"); -} - -void json_object_start(struct json_stream *result, const char *fieldname) -{ - json_start_member(result, fieldname); - result_append(result, "{"); - result_indent(result, JSMN_OBJECT); -} - -void json_object_end(struct json_stream *result) -{ - result_append(result, "\n"); - result_unindent(result, JSMN_OBJECT); - result_append_indent(result); - result_append(result, "}"); -} - void json_add_num(struct json_stream *result, const char *fieldname, unsigned int value) { - json_start_member(result, fieldname); - result_append_fmt(result, "%u", value); + json_add_member(result, fieldname, "%u", value); } void json_add_double(struct json_stream *result, const char *fieldname, double value) { - json_start_member(result, fieldname); - result_append_fmt(result, "%f", value); + json_add_member(result, fieldname, "%f", value); } void json_add_u64(struct json_stream *result, const char *fieldname, uint64_t value) { - json_start_member(result, fieldname); - result_append_fmt(result, "%"PRIu64, value); + json_add_member(result, fieldname, "%"PRIu64, value); } void json_add_literal(struct json_stream *result, const char *fieldname, const char *literal, int len) { - json_start_member(result, fieldname); - result_append_fmt(result, "%.*s", len, literal); + json_add_member(result, fieldname, "%.*s", len, literal); } void json_add_string(struct json_stream *result, const char *fieldname, const char *value) { struct json_escaped *esc = json_partial_escape(NULL, value); - json_start_member(result, fieldname); - result_append_fmt(result, "\"%s\"", esc->s); + json_add_member(result, fieldname, "\"%s\"", esc->s); tal_free(esc); } void json_add_bool(struct json_stream *result, const char *fieldname, bool value) { - json_start_member(result, fieldname); - result_append(result, value ? "true" : "false"); + json_add_member(result, fieldname, value ? "true" : "false"); } void json_add_hex(struct json_stream *result, const char *fieldname, @@ -695,33 +551,40 @@ void json_add_object(struct json_stream *result, ...) void json_add_escaped_string(struct json_stream *result, const char *fieldname, const struct json_escaped *esc TAKES) { - json_start_member(result, fieldname); - result_append_fmt(result, "\"%s\"", esc->s); + json_add_member(result, fieldname, "\"%s\"", esc->s); if (taken(esc)) tal_free(esc); } -static struct json_stream *new_json_stream(struct command *cmd) +static struct json_stream *attach_json_stream(struct command *cmd) { - struct json_stream *r = tal(cmd, struct json_stream); - - r->cmd = cmd; -#if DEVELOPER - r->wrapping = tal_arr(r, jsmntype_t, 0); -#endif - r->indent = 0; - r->empty = true; + struct json_stream *js = new_json_stream(cmd, cmd); + /* If they still care about the result, wake them */ + if (cmd->jcon) { + /* FIXME: We only allow one command at a time */ + assert(!cmd->jcon->js); + cmd->jcon->js = js; + io_wake(cmd->jcon); + } assert(!cmd->have_json_stream); cmd->have_json_stream = true; - return r; + return js; +} + +static struct json_stream *json_start(struct command *cmd) +{ + struct json_stream *js = attach_json_stream(cmd); + + json_stream_append_fmt(js, "{ \"jsonrpc\": \"2.0\", \"id\" : %s, ", + cmd->id); + return js; } struct json_stream *json_stream_success(struct command *cmd) { - struct json_stream *r; - r = new_json_stream(cmd); - result_append(r, "\"result\" : "); + struct json_stream *r = json_start(cmd); + json_stream_append(r, "\"result\" : "); return r; } @@ -729,12 +592,12 @@ struct json_stream *json_stream_fail_nodata(struct command *cmd, int code, const char *errmsg) { - struct json_stream *r = new_json_stream(cmd); + struct json_stream *r = json_start(cmd); assert(code); assert(errmsg); - result_append_fmt(r, " \"error\" : " + json_stream_append_fmt(r, " \"error\" : " "{ \"code\" : %d," " \"message\" : \"%s\"", code, errmsg); return r; @@ -746,6 +609,6 @@ struct json_stream *json_stream_fail(struct command *cmd, { struct json_stream *r = json_stream_fail_nodata(cmd, code, errmsg); - result_append(r, ", \"data\" : "); + json_stream_append(r, ", \"data\" : "); return r; } diff --git a/lightningd/json.h b/lightningd/json.h index 749f1ab71..10e5b5b2f 100644 --- a/lightningd/json.h +++ b/lightningd/json.h @@ -159,17 +159,6 @@ bool json_tok_tok(struct command *cmd, const char *name, const jsmntok_t **out); -/* Creating JSON output */ - -/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */ -void json_array_start(struct json_stream *ptr, const char *fieldname); -/* '"fieldname" : { ' or '{ ' if fieldname is NULL */ -void json_object_start(struct json_stream *ptr, const char *fieldname); -/* ' ], ' */ -void json_array_end(struct json_stream *ptr); -/* ' }, ' */ -void json_object_end(struct json_stream *ptr); - /** * json_stream_success - start streaming a successful json result. * @cmd: the command we're running. diff --git a/lightningd/json_stream.c b/lightningd/json_stream.c new file mode 100644 index 000000000..97667bef0 --- /dev/null +++ b/lightningd/json_stream.c @@ -0,0 +1,289 @@ +#include + /* To reach into io_plan: not a public header! */ + #include +#include +#include +#include +#include +#include + +struct json_stream { +#if DEVELOPER + /* tal_arr of types (JSMN_OBJECT/JSMN_ARRAY) we're enclosed in. */ + jsmntype_t *wrapping; +#endif + /* How far to indent. */ + size_t indent; + + /* True if we haven't yet put an element in current wrapping */ + bool empty; + + /* Who is writing to this buffer now; NULL if nobody is. */ + struct command *writer; + + /* Who is io_writing from this buffer now: NULL if nobody is. */ + struct io_conn *reader; + struct io_plan *(*reader_cb)(struct io_conn *conn, void *arg); + void *reader_arg; + size_t len_read; + + /* Current command's output. */ + MEMBUF(char) outbuf; +}; + +/* Realloc helper for tal membufs */ +static void *membuf_tal_realloc(struct membuf *mb, + void *rawelems, size_t newsize) +{ + char *p = rawelems; + + tal_resize(&p, newsize); + return p; +} + +struct json_stream *new_json_stream(const tal_t *ctx, struct command *writer) +{ + struct json_stream *js = tal(ctx, struct json_stream); + + js->writer = writer; + js->reader = NULL; + membuf_init(&js->outbuf, + tal_arr(js, char, 64), 64, membuf_tal_realloc); +#if DEVELOPER + js->wrapping = tal_arr(js, jsmntype_t, 0); +#endif + js->indent = 0; + js->empty = true; + return js; +} + +bool json_stream_still_writing(const struct json_stream *js) +{ + return js->writer != NULL; +} + +void json_stream_close(struct json_stream *js, struct command *writer) +{ + /* FIXME: We use writer == NULL for malformed: make writer a void *? + * I used to assert(writer); here. */ + assert(js->writer == writer); + + js->writer = NULL; +} + +/* FIXME: This, or something prettier (io_replan?) belong in ccan/io! */ +static void adjust_io_write(struct io_conn *conn, ptrdiff_t delta) +{ + conn->plan[IO_OUT].arg.u1.cp += delta; +} + +/* Make sure js->outbuf has room for len */ +static void mkroom(struct json_stream *js, size_t len) +{ + ptrdiff_t delta = membuf_prepare_space(&js->outbuf, len); + + /* If io_write is in progress, we shift it to point to new buffer pos */ + if (js->reader) + adjust_io_write(js->reader, delta); +} + +static void js_written_some(struct json_stream *js) +{ + /* Wake the stream reader. FIXME: Could have a flag here to optimize */ + io_wake(js); +} + +void json_stream_append(struct json_stream *js, const char *str) +{ + size_t len = strlen(str); + + mkroom(js, len); + memcpy(membuf_add(&js->outbuf, len), str, len); + js_written_some(js); +} + +static void json_stream_append_vfmt(struct json_stream *js, + const char *fmt, va_list ap) +{ + size_t fmtlen; + va_list ap2; + + /* Make a copy in case we need it below. */ + va_copy(ap2, ap); + + /* Try printing in place first. */ + fmtlen = vsnprintf(membuf_space(&js->outbuf), + membuf_num_space(&js->outbuf), fmt, ap); + + /* Horrible subtlety: vsnprintf *will* NUL terminate, even if it means + * chopping off the last character. So if fmtlen == + * membuf_num_space(&jcon->outbuf), the result was truncated! */ + if (fmtlen >= membuf_num_space(&js->outbuf)) { + /* Make room for NUL terminator, even though we don't want it */ + mkroom(js, fmtlen + 1); + vsprintf(membuf_space(&js->outbuf), fmt, ap2); + } + membuf_added(&js->outbuf, fmtlen); + js_written_some(js); + va_end(ap2); +} + +void PRINTF_FMT(2,3) +json_stream_append_fmt(struct json_stream *js, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + json_stream_append_vfmt(js, fmt, ap); + va_end(ap); +} + +static void check_fieldname(const struct json_stream *js, + const char *fieldname) +{ +#if DEVELOPER + size_t n = tal_count(js->wrapping); + if (n == 0) + /* Can't have a fieldname if not in anything! */ + assert(!fieldname); + else if (js->wrapping[n-1] == JSMN_ARRAY) + /* No fieldnames in arrays. */ + assert(!fieldname); + else + /* Must have fieldnames in objects. */ + assert(fieldname); +#endif +} + +static void js_append_indent(struct json_stream *js) +{ + static const char indent_buf[] = " "; + size_t len; + + for (size_t i = 0; i < js->indent * 2; i += len) { + len = js->indent * 2; + if (len > sizeof(indent_buf)-1) + len = sizeof(indent_buf)-1; + /* Use tail of indent_buf string. */ + json_stream_append(js, indent_buf + sizeof(indent_buf) - 1 - len); + } +} + +static void json_start_member(struct json_stream *js, const char *fieldname) +{ + /* Prepend comma if required. */ + if (!js->empty) + json_stream_append(js, ", \n"); + else + json_stream_append(js, "\n"); + + js_append_indent(js); + + check_fieldname(js, fieldname); + if (fieldname) + json_stream_append_fmt(js, "\"%s\": ", fieldname); + js->empty = false; +} + +static void js_indent(struct json_stream *js, jsmntype_t type) +{ +#if DEVELOPER + *tal_arr_expand(&js->wrapping) = type; +#endif + js->empty = true; + js->indent++; +} + +static void js_unindent(struct json_stream *js, jsmntype_t type) +{ + assert(js->indent); +#if DEVELOPER + assert(tal_count(js->wrapping) == js->indent); + assert(js->wrapping[js->indent-1] == type); + tal_resize(&js->wrapping, js->indent-1); +#endif + js->empty = false; + js->indent--; +} + +void json_array_start(struct json_stream *js, const char *fieldname) +{ + json_start_member(js, fieldname); + json_stream_append(js, "["); + js_indent(js, JSMN_ARRAY); +} + +void json_array_end(struct json_stream *js) +{ + json_stream_append(js, "\n"); + js_unindent(js, JSMN_ARRAY); + js_append_indent(js); + json_stream_append(js, "]"); +} + +void json_object_start(struct json_stream *js, const char *fieldname) +{ + json_start_member(js, fieldname); + json_stream_append(js, "{"); + js_indent(js, JSMN_OBJECT); +} + +void json_object_end(struct json_stream *js) +{ + json_stream_append(js, "\n"); + js_unindent(js, JSMN_OBJECT); + js_append_indent(js); + json_stream_append(js, "}"); +} + +void PRINTF_FMT(3,4) +json_add_member(struct json_stream *js, const char *fieldname, + const char *fmt, ...) +{ + va_list ap; + + json_start_member(js, fieldname); + va_start(ap, fmt); + json_stream_append_vfmt(js, fmt, ap); + va_end(ap); +} + +/* This is where we read the json_stream and write it to conn */ +static struct io_plan *json_stream_output_write(struct io_conn *conn, + struct json_stream *js) +{ + /* For when we've just done some output */ + membuf_consume(&js->outbuf, js->len_read); + + /* Get how much we can write out from js */ + js->len_read = membuf_num_elems(&js->outbuf); + + /* Nothing in buffer? */ + if (js->len_read == 0) { + /* We're not doing io_write now, unset. */ + js->reader = NULL; + if (!json_stream_still_writing(js)) + return js->reader_cb(conn, js->reader_arg); + return io_out_wait(conn, js, json_stream_output_write, js); + } + + js->reader = conn; + return io_write(conn, + membuf_elems(&js->outbuf), js->len_read, + json_stream_output_write, js); +} + +struct io_plan *json_stream_output_(struct json_stream *js, + struct io_conn *conn, + struct io_plan *(*cb)(struct io_conn *conn, + void *arg), + void *arg) +{ + assert(!js->reader); + + js->reader_cb = cb; + js->reader_arg = arg; + + js->len_read = 0; + return json_stream_output_write(conn, js); +} diff --git a/lightningd/json_stream.h b/lightningd/json_stream.h new file mode 100644 index 000000000..a79fff890 --- /dev/null +++ b/lightningd/json_stream.h @@ -0,0 +1,97 @@ +/* lightningd/json_stream.h + * Helpers for outputting JSON results into a membuf. + */ +#ifndef LIGHTNING_LIGHTNINGD_JSON_STREAM_H +#define LIGHTNING_LIGHTNINGD_JSON_STREAM_H +#include "config.h" +#include +#include +#include +#include +#include +#include +#include + +struct command; +struct io_conn; +struct json_stream; + +/** + * new_json_stream - create a new JSON stream. + * @ctx: tal context for allocation. + * @writer: object responsible for writing to this stream. + */ +struct json_stream *new_json_stream(const tal_t *ctx, struct command *writer); + +/** + * json_stream_close - finished writing to a JSON stream. + * @js: the json_stream. + * @writer: object responsible for writing to this stream. + */ +void json_stream_close(struct json_stream *js, struct command *writer); + +/** + * json_stream_still_writing - is someone currently writing to this stream? + * @js: the json_stream. + * + * Has this json_stream not been closed yet? + */ +bool json_stream_still_writing(const struct json_stream *js); + + +/* '"fieldname" : [ ' or '[ ' if fieldname is NULL */ +void json_array_start(struct json_stream *js, const char *fieldname); +/* '"fieldname" : { ' or '{ ' if fieldname is NULL */ +void json_object_start(struct json_stream *ks, const char *fieldname); +/* ' ], ' */ +void json_array_end(struct json_stream *js); +/* ' }, ' */ +void json_object_end(struct json_stream *js); + +/** + * json_stream_append - literally insert this string into the json_stream. + * @js: the json_stream. + * @str: the string. + */ +void json_stream_append(struct json_stream *js, const char *str); + +/** + * json_stream_append_fmt - insert formatted string into the json_stream. + * @js: the json_stream. + * @fmt...: the printf-style format + */ +void PRINTF_FMT(2,3) +json_stream_append_fmt(struct json_stream *js, const char *fmt, ...); + +/** + * json_add_member - add a generic member. + * @js: the json_stream. + * @fieldname: optional fieldname. + * @fmt...: the printf-style format + */ +void PRINTF_FMT(3,4) +json_add_member(struct json_stream *js, const char *fieldname, + const char *fmt, ...); + +/** + * json_stream_output - start writing out a json_stream to this conn. + * @js: the json_stream + * @conn: the io_conn to write out to. + * @cb: the callback to call once it's all written. + * @arg: the argument to @cb + */ +#define json_stream_output(js, conn, cb, arg) \ + json_stream_output_((js), (conn), \ + typesafe_cb_preargs(struct io_plan *, \ + void *, \ + (cb), (arg), \ + struct io_conn *), \ + (arg)) + +struct io_plan *json_stream_output_(struct json_stream *js, + struct io_conn *conn, + struct io_plan *(*cb)(struct io_conn *conn, + void *arg), + void *arg); + +#endif /* LIGHTNING_LIGHTNINGD_JSON_STREAM_H */ diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index c7cabe1d5..9161c97f2 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -32,16 +31,6 @@ #include #include -/* Realloc helper for tal membufs */ -static void *membuf_tal_realloc(struct membuf *mb, - void *rawelems, size_t newsize) -{ - char *p = rawelems; - - tal_resize(&p, newsize); - return p; -} - /* jcon and cmd have separate lifetimes: we detach them on either destruction */ static void destroy_jcon(struct json_connection *jcon) { @@ -54,12 +43,6 @@ static void destroy_jcon(struct json_connection *jcon) tal_free(jcon->log); } -/* FIXME: This, or something prettier (io_replan?) belong in ccan/io! */ -static void adjust_io_write(struct io_conn *conn, ptrdiff_t delta) -{ - conn->plan[IO_OUT].arg.u1.cp += delta; -} - static void json_help(struct command *cmd, const char *buffer, const jsmntok_t *params); @@ -283,57 +266,6 @@ static const struct json_command *find_cmd(const char *buffer, return NULL; } -/* Make sure jcon->outbuf has room for len */ -static void json_connection_mkroom(struct json_connection *jcon, size_t len) -{ - ptrdiff_t delta = membuf_prepare_space(&jcon->outbuf, len); - - /* If io_write is in progress, we shift it to point to new buffer pos */ - if (io_lock_taken(jcon->lock)) - adjust_io_write(jcon->conn, delta); -} - -void jcon_append(struct json_connection *jcon, const char *str) -{ - size_t len = strlen(str); - - json_connection_mkroom(jcon, len); - memcpy(membuf_add(&jcon->outbuf, len), str, len); - - /* Wake writer. */ - io_wake(jcon); -} - -void jcon_append_vfmt(struct json_connection *jcon, const char *fmt, va_list ap) -{ - size_t fmtlen; - va_list ap2; - - /* Make a copy in case we need it below. */ - va_copy(ap2, ap); - - /* Try printing in place first. */ - fmtlen = vsnprintf(membuf_space(&jcon->outbuf), - membuf_num_space(&jcon->outbuf), fmt, ap); - - /* Horrible subtlety: vsnprintf *will* NUL terminate, even if it means - * chopping off the last character. So if fmtlen == - * membuf_num_space(&jcon->outbuf), the result was truncated! */ - if (fmtlen < membuf_num_space(&jcon->outbuf)) { - membuf_added(&jcon->outbuf, fmtlen); - } else { - /* Make room for NUL terminator, even though we don't want it */ - json_connection_mkroom(jcon, fmtlen + 1); - vsprintf(membuf_space(&jcon->outbuf), fmt, ap2); - membuf_added(&jcon->outbuf, fmtlen); - } - - va_end(ap2); - - /* Wake writer. */ - io_wake(jcon); -} - struct json_stream *null_response(struct command *cmd) { struct json_stream *response; @@ -357,27 +289,34 @@ static void destroy_command(struct command *cmd) cmd->jcon->command = NULL; } -/* FIXME: Remove result arg here! */ void command_success(struct command *cmd, struct json_stream *result) { assert(cmd); assert(cmd->have_json_stream); - if (cmd->jcon) - jcon_append(cmd->jcon, " }\n\n"); + json_stream_append(result, " }\n\n"); + json_stream_close(result, cmd); if (cmd->ok) *(cmd->ok) = true; + + /* If we have a jcon, it will free result for us. */ + if (cmd->jcon) + tal_steal(cmd->jcon, result); + tal_free(cmd); } -/* FIXME: Remove result arg here! */ void command_failed(struct command *cmd, struct json_stream *result) { assert(cmd->have_json_stream); /* Have to close error */ - if (cmd->jcon) - jcon_append(cmd->jcon, " } }\n\n"); + json_stream_append(result, " } }\n\n"); + json_stream_close(result, cmd); if (cmd->ok) *(cmd->ok) = false; + /* If we have a jcon, it will free result for us. */ + if (cmd->jcon) + tal_steal(cmd->jcon, result); + tal_free(cmd); } @@ -403,23 +342,23 @@ void command_still_pending(struct command *cmd) cmd->pending = true; } -static void jcon_start(struct json_connection *jcon, const char *id) -{ - jcon_append(jcon, "{ \"jsonrpc\": \"2.0\", \"id\" : "); - jcon_append(jcon, id); - jcon_append(jcon, ", "); -} - static void json_command_malformed(struct json_connection *jcon, const char *id, const char *error) { - jcon_start(jcon, id); - jcon_append(jcon, - tal_fmt(tmpctx, " \"error\" : " - "{ \"code\" : %d," - " \"message\" : \"%s\" } }\n\n", - JSONRPC2_INVALID_REQUEST, error)); + /* FIXME: We only allow one command at a time */ + assert(!jcon->js); + jcon->js = new_json_stream(jcon, NULL); + io_wake(jcon); + + json_stream_append_fmt(jcon->js, + "{ \"jsonrpc\": \"2.0\", \"id\" : %s," + " \"error\" : " + "{ \"code\" : %d," + " \"message\" : \"%s\" } }\n\n", + id, JSONRPC2_INVALID_REQUEST, error); + + json_stream_close(jcon->js, NULL); } /* Returns true if command already completed. */ @@ -463,9 +402,6 @@ static bool parse_request(struct json_connection *jcon, const jsmntok_t tok[]) jcon->command = c; tal_add_destructor(c, destroy_command); - /* Write start of response: rest will be appended directly. */ - jcon_start(jcon, c->id); - if (!method || !params) { command_fail(c, JSONRPC2_INVALID_REQUEST, method ? "No params" : "No method"); @@ -506,19 +442,28 @@ static bool parse_request(struct json_connection *jcon, const jsmntok_t tok[]) } /* Mutual recursion */ -static struct io_plan *locked_write_json(struct io_conn *conn, - struct json_connection *jcon); -static struct io_plan *write_json(struct io_conn *conn, - struct json_connection *jcon); +static struct io_plan *stream_out_complete(struct io_conn *conn, + struct json_connection *jcon); -static struct io_plan *write_json_done(struct io_conn *conn, - struct json_connection *jcon) +static struct io_plan *start_json_stream(struct io_conn *conn, + struct json_connection *jcon) { - membuf_consume(&jcon->outbuf, jcon->out_amount); + /* If something has created an output buffer, start streaming. */ + if (jcon->js) + return json_stream_output(jcon->js, conn, + stream_out_complete, jcon); - /* If we have more to write, do it now. */ - if (membuf_num_elems(&jcon->outbuf)) - return write_json(conn, jcon); + /* Wait for attach_json_stream */ + return io_out_wait(conn, jcon, start_json_stream, jcon); +} + +/* Command has completed writing, and we've written it all out to conn. */ +static struct io_plan *stream_out_complete(struct io_conn *conn, + struct json_connection *jcon) +{ + /* Free up the json_stream for next command. */ + assert(jcon->js); + jcon->js = tal_free(jcon->js); if (jcon->stop) { log_unusual(jcon->log, "JSON-RPC shutdown"); @@ -527,29 +472,11 @@ static struct io_plan *write_json_done(struct io_conn *conn, return io_close(conn); } - /* If command is done and we've output everything, wake read_json - * for next command. */ - if (!jcon->command) - io_wake(conn); + /* Tell reader to run next command. */ + io_wake(conn); - io_lock_release(jcon->lock); /* Wait for more output. */ - return io_out_wait(conn, jcon, locked_write_json, jcon); -} - -static struct io_plan *write_json(struct io_conn *conn, - struct json_connection *jcon) -{ - jcon->out_amount = membuf_num_elems(&jcon->outbuf); - return io_write(conn, - membuf_elems(&jcon->outbuf), jcon->out_amount, - write_json_done, jcon); -} - -static struct io_plan *locked_write_json(struct io_conn *conn, - struct json_connection *jcon) -{ - return io_lock_acquire_out(conn, jcon->lock, write_json, jcon); + return start_json_stream(conn, jcon); } static struct io_plan *read_json(struct io_conn *conn, @@ -567,6 +494,12 @@ static struct io_plan *read_json(struct io_conn *conn, if (jcon->used == tal_count(jcon->buffer)) tal_resize(&jcon->buffer, jcon->used * 2); + /* We wait for pending output to be consumed, to avoid DoS */ + if (jcon->js) { + jcon->len_read = 0; + return io_wait(conn, conn, read_json, jcon); + } + toks = json_parse_input(jcon->buffer, jcon->used, &valid); if (!toks) { if (!valid) { @@ -629,9 +562,7 @@ static struct io_plan *jcon_connected(struct io_conn *conn, jcon->used = 0; jcon->buffer = tal_arr(jcon, char, 64); jcon->stop = false; - jcon->lock = io_lock_new(jcon); - membuf_init(&jcon->outbuf, - tal_arr(jcon, char, 64), 64, membuf_tal_realloc); + jcon->js = NULL; jcon->len_read = 0; jcon->command = NULL; @@ -650,7 +581,7 @@ static struct io_plan *jcon_connected(struct io_conn *conn, * pattern would break down, so we use this trick. */ return io_duplex(conn, read_json(conn, jcon), - io_out_wait(conn, jcon, locked_write_json, jcon)); + start_json_stream(conn, jcon)); } static struct io_plan *incoming_jcon_connected(struct io_conn *conn, diff --git a/lightningd/jsonrpc.h b/lightningd/jsonrpc.h index aa6a28057..46e366f36 100644 --- a/lightningd/jsonrpc.h +++ b/lightningd/jsonrpc.h @@ -3,10 +3,9 @@ #include "config.h" #include #include -#include -#include #include #include +#include #include struct bitcoin_txid; @@ -68,12 +67,8 @@ struct json_connection { /* Current command. */ struct command *command; - /* Current command's output. */ - MEMBUF(char) outbuf; - - /* How much we're writing right now. */ - size_t out_amount; - struct io_lock *lock; + /* Our json_stream */ + struct json_stream *js; }; struct json_command { @@ -94,10 +89,6 @@ void PRINTF_FMT(3, 4) command_fail(struct command *cmd, int code, /* Mainly for documentation, that we plan to close this later. */ void command_still_pending(struct command *cmd); -/* Low level jcon routines. */ -void jcon_append(struct json_connection *jcon, const char *str); -void jcon_append_vfmt(struct json_connection *jcon, const char *fmt, va_list ap); - /* For initialization */ void setup_jsonrpc(struct lightningd *ld, const char *rpc_filename); diff --git a/lightningd/test/run-invoice-select-inchan.c b/lightningd/test/run-invoice-select-inchan.c index f5c9e6e25..72b3303d4 100644 --- a/lightningd/test/run-invoice-select-inchan.c +++ b/lightningd/test/run-invoice-select-inchan.c @@ -145,19 +145,19 @@ void json_add_uncommitted_channel(struct json_stream *response UNNEEDED, const struct uncommitted_channel *uc UNNEEDED) { fprintf(stderr, "json_add_uncommitted_channel called!\n"); abort(); } /* Generated stub for json_array_end */ -void json_array_end(struct json_stream *ptr UNNEEDED) +void json_array_end(struct json_stream *js UNNEEDED) { fprintf(stderr, "json_array_end called!\n"); abort(); } /* Generated stub for json_array_start */ -void json_array_start(struct json_stream *ptr UNNEEDED, const char *fieldname UNNEEDED) +void json_array_start(struct json_stream *js UNNEEDED, const char *fieldname UNNEEDED) { fprintf(stderr, "json_array_start called!\n"); abort(); } /* Generated stub for json_escape */ struct json_escaped *json_escape(const tal_t *ctx UNNEEDED, const char *str TAKES UNNEEDED) { fprintf(stderr, "json_escape called!\n"); abort(); } /* Generated stub for json_object_end */ -void json_object_end(struct json_stream *ptr UNNEEDED) +void json_object_end(struct json_stream *js UNNEEDED) { fprintf(stderr, "json_object_end called!\n"); abort(); } /* Generated stub for json_object_start */ -void json_object_start(struct json_stream *ptr UNNEEDED, const char *fieldname UNNEEDED) +void json_object_start(struct json_stream *ks UNNEEDED, const char *fieldname UNNEEDED) { fprintf(stderr, "json_object_start called!\n"); abort(); } /* Generated stub for json_stream_fail */ struct json_stream *json_stream_fail(struct command *cmd UNNEEDED, diff --git a/lightningd/test/run-jsonrpc.c b/lightningd/test/run-jsonrpc.c index 91fc71855..ad15a5315 100644 --- a/lightningd/test/run-jsonrpc.c +++ b/lightningd/test/run-jsonrpc.c @@ -1,4 +1,5 @@ #include "../json_escaped.c" +#include "../json_stream.c" #include "../jsonrpc.c" #include "../json.c" @@ -61,9 +62,7 @@ bool deprecated_apis; static int test_json_filter(void) { - struct command *cmd = talz(NULL, struct command); - struct json_connection *jcon = talz(cmd, struct json_connection); - struct json_stream *result = json_stream_success(cmd); + struct json_stream *result = new_json_stream(NULL, NULL); jsmntok_t *toks; const jsmntok_t *x; bool valid; @@ -71,12 +70,6 @@ static int test_json_filter(void) char *badstr = tal_arr(result, char, 256); const char *str; - /* We need to initialize membuf so we can gather results. */ - cmd->jcon = jcon; - jcon->lock = io_lock_new(jcon); - membuf_init(&jcon->outbuf, - tal_arr(cmd, char, 64), 64, membuf_tal_realloc); - /* Fill with junk, and nul-terminate (256 -> 0) */ for (i = 1; i < 257; i++) badstr[i-1] = i; @@ -86,8 +79,8 @@ static int test_json_filter(void) json_object_end(result); /* Parse back in, make sure nothing crazy. */ - str = tal_strndup(cmd, membuf_elems(&jcon->outbuf), - membuf_num_elems(&jcon->outbuf)); + str = tal_strndup(result, membuf_elems(&result->outbuf), + membuf_num_elems(&result->outbuf)); toks = json_parse_input(str, strlen(str), &valid); assert(valid); @@ -105,7 +98,7 @@ static int test_json_filter(void) assert((unsigned)str[i] >= ' '); assert((unsigned)str[i] != 127); } - tal_free(cmd); + tal_free(result); return 0; } @@ -115,27 +108,19 @@ static void test_json_escape(void) for (i = 1; i < 256; i++) { char badstr[2]; - struct command *cmd = talz(NULL, struct command); - struct json_connection *jcon = talz(cmd, struct json_connection); - struct json_stream *result = json_stream_success(cmd); + struct json_stream *result = new_json_stream(NULL, NULL); struct json_escaped *esc; badstr[0] = i; badstr[1] = 0; - /* We need to initialize membuf so we can gather results. */ - cmd->jcon = jcon; - jcon->lock = io_lock_new(jcon); - membuf_init(&jcon->outbuf, - tal_arr(cmd, char, 64), 64, membuf_tal_realloc); - json_object_start(result, NULL); esc = json_escape(NULL, badstr); json_add_escaped_string(result, "x", take(esc)); json_object_end(result); - const char *str = tal_strndup(cmd, membuf_elems(&jcon->outbuf), - membuf_num_elems(&jcon->outbuf)); + const char *str = tal_strndup(result, membuf_elems(&result->outbuf), + membuf_num_elems(&result->outbuf)); if (i == '\\' || i == '"' || i == '\n' || i == '\r' || i == '\b' || i == '\t' || i == '\f') @@ -147,7 +132,7 @@ static void test_json_escape(void) expect[11] = i; assert(streq(str, expect)); } - tal_free(cmd); + tal_free(result); } } diff --git a/lightningd/test/run-param.c b/lightningd/test/run-param.c index 770872482..d7f652222 100644 --- a/lightningd/test/run-param.c +++ b/lightningd/test/run-param.c @@ -1,5 +1,7 @@ +#include "config.h" #include "../json.c" #include "../json_escaped.c" +#include "../json_stream.c" #include "../param.c" #include #include @@ -40,12 +42,6 @@ const char *feerate_name(enum feerate feerate UNNEEDED) /* Generated stub for fmt_wireaddr_without_port */ char *fmt_wireaddr_without_port(const tal_t *ctx UNNEEDED, const struct wireaddr *a UNNEEDED) { fprintf(stderr, "fmt_wireaddr_without_port called!\n"); abort(); } -/* Generated stub for jcon_append */ -void jcon_append(struct json_connection *jcon UNNEEDED, const char *str UNNEEDED) -{ fprintf(stderr, "jcon_append called!\n"); abort(); } -/* Generated stub for jcon_append_vfmt */ -void jcon_append_vfmt(struct json_connection *jcon UNNEEDED, const char *fmt UNNEEDED, va_list ap UNNEEDED) -{ fprintf(stderr, "jcon_append_vfmt called!\n"); abort(); } /* Generated stub for json_feerate_estimate */ bool json_feerate_estimate(struct command *cmd UNNEEDED, u32 **feerate_per_kw UNNEEDED, enum feerate feerate UNNEEDED) diff --git a/wallet/test/run-wallet.c b/wallet/test/run-wallet.c index d21bd838a..514028b89 100644 --- a/wallet/test/run-wallet.c +++ b/wallet/test/run-wallet.c @@ -219,20 +219,20 @@ void json_add_uncommitted_channel(struct json_stream *response UNNEEDED, const struct uncommitted_channel *uc UNNEEDED) { fprintf(stderr, "json_add_uncommitted_channel called!\n"); abort(); } /* Generated stub for json_array_end */ -void json_array_end(struct json_stream *ptr UNNEEDED) +void json_array_end(struct json_stream *js UNNEEDED) { fprintf(stderr, "json_array_end called!\n"); abort(); } /* Generated stub for json_array_start */ -void json_array_start(struct json_stream *ptr UNNEEDED, const char *fieldname UNNEEDED) +void json_array_start(struct json_stream *js UNNEEDED, const char *fieldname UNNEEDED) { fprintf(stderr, "json_array_start called!\n"); abort(); } /* Generated stub for json_escaped_string_ */ struct json_escaped *json_escaped_string_(const tal_t *ctx UNNEEDED, const void *bytes UNNEEDED, size_t len UNNEEDED) { fprintf(stderr, "json_escaped_string_ called!\n"); abort(); } /* Generated stub for json_object_end */ -void json_object_end(struct json_stream *ptr UNNEEDED) +void json_object_end(struct json_stream *js UNNEEDED) { fprintf(stderr, "json_object_end called!\n"); abort(); } /* Generated stub for json_object_start */ -void json_object_start(struct json_stream *ptr UNNEEDED, const char *fieldname UNNEEDED) +void json_object_start(struct json_stream *ks UNNEEDED, const char *fieldname UNNEEDED) { fprintf(stderr, "json_object_start called!\n"); abort(); } /* Generated stub for json_stream_success */ struct json_stream *json_stream_success(struct command *cmd UNNEEDED)