ccan: update and import rbuf module.

This is a simple helper for dealing with buffered I/O.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Makefile

@@ -77,6 +77,7 @@ CCAN_OBJS :=					\
 	ccan-opt.o				\
 	ccan-pipecmd.o				\
 	ccan-ptr_valid.o			\
+	ccan-rbuf.o				\
 	ccan-read_write_all.o			\
 	ccan-str-hex.o				\
 	ccan-str.o				\
@@ -134,6 +135,7 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/pipecmd/pipecmd.h		\
 	$(CCANDIR)/ccan/ptr_valid/ptr_valid.h		\
 	$(CCANDIR)/ccan/ptrint/ptrint.h			\
+	$(CCANDIR)/ccan/rbuf/rbuf.h			\
 	$(CCANDIR)/ccan/read_write_all/read_write_all.h	\
 	$(CCANDIR)/ccan/short_types/short_types.h	\
 	$(CCANDIR)/ccan/str/hex/hex.h			\
@@ -545,3 +547,5 @@ ccan-fdpass.o: $(CCANDIR)/ccan/fdpass/fdpass.c
 	$(CC) $(CFLAGS) -c -o $@ $<
 ccan-bitops.o: $(CCANDIR)/ccan/bitops/bitops.c
 	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-rbuf.o: $(CCANDIR)/ccan/rbuf/rbuf.c
+	$(CC) $(CFLAGS) -c -o $@ $<

ccan/ccan/rbuf/LICENSE

@@ -0,0 +1 @@
+../../licenses/BSD-MIT

ccan/ccan/rbuf/_info

@@ -0,0 +1,53 @@
+#include "config.h"
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * rbuf - buffered I/O input primitive.
+ *
+ * This code is like stdio, only simpler and more transparent to the user.
+ *
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ * License: BSD-MIT
+ *
+ * Example:
+ *	#include <ccan/rbuf/rbuf.h>
+ *	#include <ccan/err/err.h>
+ *	#include <stdlib.h>
+ *	#include <unistd.h>
+ *
+ *	// Dumb demo program to replace ' ' with '*'.
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct rbuf in;
+ *		char *word;
+ *
+ *		if (argv[1]) {
+ *			if (!rbuf_open(&in, argv[1], NULL, 0))
+ *				err(1, "Failed opening %s", argv[1]);
+ *		} else
+ *			rbuf_init(&in, STDIN_FILENO, NULL, 0);
+ *
+ *		while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL)
+ *			printf("%s*", word);
+ *
+ *		if (errno)
+ *			err(1, "Reading %s", argv[1] ? argv[1] : "<stdin>");
+ *
+ *		// Free the buffer, just because we can.
+ *		free(in.buf);
+ *		return 0;
+ *	}
+ */
+int main(int argc, char *argv[])
+{
+	/* Expect exactly one argument */
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		return 0;
+	}
+
+	return 1;
+}

ccan/ccan/rbuf/rbuf.c

@@ -0,0 +1,130 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#include <ccan/rbuf/rbuf.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max)
+{
+	int fd = open(name, O_RDONLY);
+	if (fd >= 0) {
+		rbuf_init(rbuf, fd, buf, buf_max);
+		return true;
+	}
+	return false;
+}
+
+static size_t rem(const struct rbuf *buf)
+{
+	return buf->buf_end - (buf->start + buf->len);
+}
+
+size_t rbuf_good_size(int fd)
+{
+	struct stat st;
+
+	if (fstat(fd, &st) == 0 && st.st_blksize >= 4096)
+		return st.st_blksize;
+	return 4096;
+}
+
+static bool enlarge_buf(struct rbuf *buf, size_t len,
+			void *(*resize)(void *buf, size_t len))
+{
+	char *new;
+	if (!resize) {
+		errno = ENOMEM;
+		return false;
+	}
+	if (!len)
+		len = rbuf_good_size(buf->fd);
+	new = resize(buf->buf, len);
+	if (!new)
+		return false;
+	buf->start += (new - buf->buf);
+	buf->buf = new;
+	buf->buf_end = new + len;
+	return true;
+}
+
+static ssize_t get_more(struct rbuf *rbuf,
+			void *(*resize)(void *buf, size_t len))
+{
+	size_t r;
+
+	if (rbuf->start + rbuf->len == rbuf->buf_end) {
+		if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize))
+			return -1;
+	}
+
+	r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf));
+	if (r <= 0)
+		return r;
+
+	rbuf->len += r;
+	return r;
+}
+
+void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+{
+	ssize_t r;
+
+	/* Move back to start of buffer if we're empty. */
+	if (!rbuf->len)
+		rbuf->start = rbuf->buf;
+
+	while ((r = get_more(rbuf, resize)) != 0)
+		if (r < 0)
+			return NULL;
+	return rbuf->start;
+}
+
+void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len))
+{
+	if (!rbuf->len) {
+		rbuf->start = rbuf->buf;
+		if (get_more(rbuf, resize) < 0)
+			return NULL;
+	}
+	return rbuf->start;
+}
+
+char *rbuf_read_str(struct rbuf *rbuf, char term,
+		    void *(*resize)(void *buf, size_t len))
+{
+	char *p, *ret;
+	ssize_t r = 0;
+	size_t prev = 0;
+
+	/* Move back to start of buffer if we're empty. */
+	if (!rbuf->len)
+		rbuf->start = rbuf->buf;
+
+	while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) {
+		prev += r;
+		r = get_more(rbuf, resize);
+		if (r < 0)
+			return NULL;
+		/* EOF with no term. */
+		if (r == 0) {
+			/* Nothing read at all? */
+			if (!rbuf->len && term) {
+				errno = 0;
+				return NULL;
+			}
+			/* Put term after input (get_more made room). */
+			assert(rbuf->start + rbuf->len < rbuf->buf_end);
+			rbuf->start[rbuf->len] = '\0';
+			ret = rbuf->start;
+			rbuf_consume(rbuf, rbuf->len);
+			return ret;
+		}
+	}
+	*p = '\0';
+	ret = rbuf->start;
+	rbuf_consume(rbuf, p + 1 - ret);
+	return ret;
+}

ccan/ccan/rbuf/rbuf.h

@@ -0,0 +1,156 @@
+/* Licensed under BSD-MIT - see LICENSE file for details */
+#ifndef CCAN_RBUF_H
+#define CCAN_RBUF_H
+#include <stdio.h> // For size_t
+#include <limits.h> // For UCHAR_MAX
+#include <assert.h>
+#include <stdbool.h>
+
+struct rbuf {
+	int fd;
+
+	/* Where to read next. */
+	char *start;
+	/* How much of what is there is valid. */
+	size_t len;
+
+	/* The entire buffer memory we have to work with. */
+	char *buf, *buf_end;
+};
+
+/**
+ * rbuf_init - set up a buffer.
+ * @buf: the struct rbuf.
+ * @fd: the file descriptor.
+ * @buf: the buffer to use.
+ * @buf_max: the size of the buffer.
+ */
+static inline void rbuf_init(struct rbuf *buf,
+			     int fd, char *buffer, size_t buf_max)
+{
+	buf->fd = fd;
+	buf->start = buf->buf = buffer;
+	buf->len = 0;
+	buf->buf_end = buffer + buf_max;
+}
+
+/**
+ * rbuf_open - set up a buffer by opening a file.
+ * @buf: the struct rbuf.
+ * @filename: the filename
+ * @buf: the buffer to use.
+ * @buf_max: the size of the buffer.
+ *
+ * Returns false if the open fails.  If @buf_max is 0, then the buffer
+ * will be resized to rbuf_good_size() on first rbuf_fill.
+ *
+ * Example:
+ *	struct rbuf in;
+ *
+ *	if (!rbuf_open(&in, "foo", NULL, 0))
+ *		err(1, "Could not open foo");
+ */
+bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max);
+
+/**
+ * rbuf_good_size - get a good buffer size for this fd.
+ * @fd: the file descriptor.
+ *
+ * If you don't know what size you want, try this.
+ */
+size_t rbuf_good_size(int fd);
+
+/**
+ * rbuf_fill - read into a buffer if it's empty.
+ * @buf: the struct rbuf
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read will
+ * return NULL (with errno set to ENOMEM).  If a read fails, then NULL
+ * is also returned.  If there is nothing more to read, it will return
+ * NULL with errno set to 0.  Otherwise, returns @buf->start; @buf->len
+ * is the valid length of the buffer.
+ *
+ * You need to call rbuf_consume() to mark data in the buffer as
+ * consumed.
+ *
+ * Example:
+ *	while (rbuf_fill(&in, realloc)) {
+ *		printf("%.*s\n", (int)in.len, in.start);
+ *		rbuf_consume(&in, in.len);
+ *	}
+ *	if (errno)
+ *		err(1, "reading foo");
+ */
+void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+
+/**
+ * rbuf_consume - helper to use up data in a buffer.
+ * @buf: the struct rbuf
+ * @len: the length (from @buf->start) you used.
+ *
+ * After rbuf_fill() you should indicate the data you've used with
+ * rbuf_consume().  That way rbuf_fill() will know if it has anything
+ * to do.
+ */
+static inline void rbuf_consume(struct rbuf *buf, size_t len)
+{
+	buf->len -= len;
+	buf->start += len;
+}
+
+/**
+ * rbuf_fill_all - read rest of file into a buffer.
+ * @buf: the struct rbuf
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read_all
+ * will return NULL (with errno set to ENOMEM).  If a read fails,
+ * then NULL is also returned, otherwise returns @buf->start.
+ *
+ * Example:
+ *	if (!rbuf_fill_all(&in, realloc)) {
+ *		if (errno)
+ *			err(1, "reading foo");
+ *	}
+ */
+void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len));
+
+/**
+ * rbuf_read_str - fill into a buffer up to a terminator, and consume string.
+ * @buf: the struct rbuf
+ * @term: the character to terminate the read.
+ * @resize: the call to resize the buffer.
+ *
+ * If @resize is needed and is NULL, or returns false, rbuf_read_str
+ * will return NULL (with errno set to ENOMEM).  If a read fails,
+ * then NULL is also returned, otherwise the next string.  It
+ * replaces the terminator @term (if any) with NUL, otherwise NUL
+ * is placed after EOF.  If you need to, you can tell this has happened
+ * because the nul terminator will be at @buf->start (normally it will
+ * be at @buf->start - 1).
+ *
+ * If there is nothing remaining to be read, NULL is returned with
+ * errno set to 0, unless @term is NUL, in which case it returns the
+ * empty string.
+ *
+ * Note: using @term set to NUL is a cheap way of getting an entire
+ * file into a C string, as long as the file doesn't contain NUL.
+ *
+ * Example:
+ *	char *line;
+ *
+ *	line = rbuf_read_str(&in, '\n', realloc);
+ *	if (!line) {
+ *		if (errno)
+ *			err(1, "reading foo");
+ *		else
+ *			printf("Empty file\n");
+ *	} else
+ *		printf("First line is %s\n", line);
+ *
+ */
+char *rbuf_read_str(struct rbuf *rbuf, char term,
+		    void *(*resize)(void *buf, size_t len));
+
+#endif /* CCAN_RBUF_H */

ccan/ccan/rbuf/test/run-all.c

@@ -0,0 +1,49 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	struct rbuf in;
+	char buf[4096];
+	int i, size, fd = open("run-all-file", O_WRONLY|O_CREAT, 0600);
+
+	/* This is how many tests you plan to run */
+	plan_tests(8);
+
+	/* Make sure we're bigger than a single buffer! */
+	size = rbuf_good_size(fd)*2;
+	for (i = 0; i * sizeof(buf) < size; i++) {
+		memset(buf, 0x42 + i, sizeof(buf));
+		write(fd, buf, sizeof(buf));
+	}
+	close(fd);
+
+	ok1(rbuf_open(&in, "run-all-file", NULL, 0));
+	/* Can't fill without realloc. */
+	ok1(!rbuf_fill(&in, NULL));
+	ok1(errno == ENOMEM);
+	ok1(rbuf_fill(&in, realloc));
+	/* But can't load in whole file. */
+	ok1(!rbuf_fill_all(&in, NULL));
+	ok1(errno == ENOMEM);
+	ok1(rbuf_fill_all(&in, realloc));
+	ok1(in.len == size);
+	for (i = 0; i * sizeof(buf) < size; i++) {
+		memset(buf, 0x42 + i, sizeof(buf));
+		if (memcmp(buf, in.start, sizeof(buf)) != 0) {
+			fail("Bad buffer contents");
+			break;
+		}
+		rbuf_consume(&in, sizeof(buf));
+	}
+	free(in.buf);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/rbuf/test/run-open.c

@@ -0,0 +1,25 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	struct rbuf in;
+
+	/* This is how many tests you plan to run */
+	plan_tests(5);
+
+	ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0));
+	ok1(errno == ENOENT);
+	ok1(rbuf_open(&in, "test/run-open.c", NULL, 0));
+	ok1(close(in.fd) == 0);
+	/* If this fails to stat, it should fall back */
+	ok1(rbuf_good_size(in.fd) == 4096);
+
+	return exit_status();
+}

ccan/ccan/rbuf/test/run-partial-read.c

@@ -0,0 +1,67 @@
+#include <unistd.h>
+
+static ssize_t partial_read(int fd, void *buf, size_t count)
+{
+	return read(fd, buf, 1);
+}
+static ssize_t full_read(int fd, void *buf, size_t count)
+{
+	return read(fd, buf, count);
+}
+#define read partial_read
+
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	struct rbuf in;
+	char buf[4096];
+	char *lines[100], *p;
+	int i, fd = open("test/run.c", O_RDONLY);
+
+	/* This is how many tests you plan to run */
+	plan_tests(140);
+
+	/* Grab ourselves for comparison. */
+	buf[full_read(fd, buf, sizeof(buf))] = '\0';
+	lseek(fd, SEEK_SET, 0);
+
+	for (i = 0, p = buf; *p; i++) {
+		lines[i] = p;
+		p = strchr(p, '\n');
+		*p = '\0';
+		p++;
+	}
+	lines[i] = NULL;
+
+	rbuf_init(&in, fd, malloc(31), 31);
+	ok1(in.fd == fd);
+	ok1(in.buf_end - in.buf == 31);
+	p = rbuf_read_str(&in, '\n', NULL);
+	ok1(p);
+	ok1(strcmp(p, lines[0]) == 0);
+
+	p = rbuf_read_str(&in, '\n', realloc);
+	ok1(p);
+	ok1(strcmp(p, lines[1]) == 0);
+
+	for (i = 2; lines[i]; i++) {
+		ok1(p = rbuf_read_str(&in, '\n', realloc));
+		ok1(strcmp(p, lines[i]) == 0);
+	}
+
+	p = rbuf_read_str(&in, '\n', realloc);
+	ok1(errno == 0);
+	ok1(p == NULL);
+	free(in.buf);
+
+	/* This exits depending on whether all tests passed */
+	return exit_status();
+}

ccan/ccan/rbuf/test/run-term-eof.c

@@ -0,0 +1,44 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	struct rbuf in;
+	char buf[4096], *p;
+	int fd = open("test/run-term-eof.c", O_RDONLY), len;
+
+	/* This is how many tests you plan to run */
+	plan_tests(6);
+
+	/* Grab ourselves for comparison. */
+	len = read(fd, buf, sizeof(buf));
+	buf[len] = '\0';
+	lseek(fd, SEEK_SET, 0);
+
+	/* We have exact-size buffer, which causes problems adding term. */
+	rbuf_init(&in, fd, malloc(len), len);
+	p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */
+	ok1(errno == ENOMEM);
+	ok1(!p);
+	/* This should succeed... */
+	p = rbuf_read_str(&in, 64, realloc);
+	ok1(p);
+	ok1(strcmp(p, buf) == 0);
+	free(in.buf);
+
+	/* Try again. */
+	lseek(fd, SEEK_SET, 0);
+	rbuf_init(&in, fd, malloc(len), len);
+	p = rbuf_read_str(&in, 64, realloc);
+	ok1(p);
+	ok1(strcmp(p, buf) == 0);
+	free(in.buf);
+
+	return exit_status();
+}

ccan/ccan/rbuf/test/run.c

@@ -0,0 +1,68 @@
+#include <ccan/rbuf/rbuf.h>
+/* Include the C files directly. */
+#include <ccan/rbuf/rbuf.c>
+#include <ccan/tap/tap.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	struct rbuf in;
+	char buf[4096];
+	char *lines[100], *p;
+	int i, fd = open("test/run.c", O_RDONLY), len;
+
+	/* This is how many tests you plan to run */
+	plan_tests(144);
+
+	/* Grab ourselves for comparison. */
+	len = read(fd, buf, sizeof(buf));
+	buf[len] = '\0';
+	lseek(fd, SEEK_SET, 0);
+
+	for (i = 0, p = buf; *p; i++) {
+		lines[i] = p;
+		p = strchr(p, '\n');
+		*p = '\0';
+		p++;
+	}
+	lines[i] = NULL;
+
+	rbuf_init(&in, fd, malloc(31), 31);
+	ok1(in.fd == fd);
+	ok1(in.buf_end - in.buf == 31);
+	p = rbuf_read_str(&in, '\n', NULL);
+	ok1(p);
+	ok1(strcmp(p, lines[0]) == 0);
+
+	p = rbuf_read_str(&in, '\n', realloc);
+	ok1(p);
+	ok1(strcmp(p, lines[1]) == 0);
+
+	for (i = 2; lines[i]; i++) {
+		ok1(p = rbuf_read_str(&in, '\n', realloc));
+		ok1(strcmp(p, lines[i]) == 0);
+	}
+
+	p = rbuf_read_str(&in, '\n', realloc);
+	ok1(errno == 0);
+	ok1(p == NULL);
+	free(in.buf);
+
+	/* Another way of reading the entire (text) file. */
+	lseek(fd, SEEK_SET, 0);
+	rbuf_init(&in, fd, NULL, 0);
+	p = rbuf_read_str(&in, 0, realloc);
+	ok1(p);
+	ok1(strlen(p) == len);
+
+	close(fd);
+	p = rbuf_read_str(&in, 0, realloc);
+	ok1(errno == EBADF);
+	ok1(!p);
+	free(in.buf);
+
+	return exit_status();
+}