From d7ff74e8f682ac132e03517cefb7d537e0df9c35 Mon Sep 17 00:00:00 2001
From: Christian Decker <decker.christian@gmail.com>
Date: Wed, 26 Aug 2020 13:33:23 +0200
Subject: [PATCH] json: Only start parsing once we're sure we have a message

PR #3957 improved performance considerably, however we still look over the
entire message for the message separator. If instead we just look in the
incrementally read data, we remove the quadratic behavior for large messages.

This is safe since we then loop over the messages which would drain any
message separator from the buffer before we attempt the next read.

Changelog-Fixed: bcli: Significant speedups for block synchronization
---
 lightningd/plugin.c | 42 +++++++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/lightningd/plugin.c b/lightningd/plugin.c
index a6db1b75a..1f696a2ec 100644
--- a/lightningd/plugin.c
+++ b/lightningd/plugin.c
@@ -491,30 +491,42 @@ static struct io_plan *plugin_read_json(struct io_conn *conn,
 					struct plugin *plugin)
 {
 	bool success;
+	bool have_full;
 
 	log_io(plugin->log, LOG_IO_IN, NULL, "",
 	       plugin->buffer + plugin->used, plugin->len_read);
 
+	/* Our JSON parser is pretty good at incremental parsing, but
+	 * `getrawblock` gives a giant 2MB token, which forces it to re-parse
+	 * every time until we have all of it. However, we can't complete a
+	 * JSON object without a '}', so we do a cheaper check here.
+	 */
+	have_full = memchr(plugin->buffer + plugin->used, '}',
+			   plugin->len_read);
+
 	plugin->used += plugin->len_read;
 	if (plugin->used == tal_count(plugin->buffer))
 		tal_resize(&plugin->buffer, plugin->used * 2);
 
 	/* Read and process all messages from the connection */
-	do {
-		bool destroyed;
-		const char *err;
-		err = plugin_read_json_one(plugin, &success, &destroyed);
-
-		/* If it's destroyed, conn is already freed! */
-		if (destroyed)
-			return io_close(NULL);
-
-		if (err) {
-			plugin_kill(plugin, err);
-			/* plugin_kill frees plugin */
-			return io_close(NULL);
-		}
-	} while (success);
+	if (have_full) {
+		do {
+			bool destroyed;
+			const char *err;
+			err =
+			    plugin_read_json_one(plugin, &success, &destroyed);
+
+			/* If it's destroyed, conn is already freed! */
+			if (destroyed)
+				return io_close(NULL);
+
+			if (err) {
+				plugin_kill(plugin, err);
+				/* plugin_kill frees plugin */
+				return io_close(NULL);
+			}
+		} while (success);
+	}
 
 	/* Now read more from the connection */
 	return io_read_partial(plugin->stdout_conn,