diff --git a/Makefile b/Makefile index df30067da..253c61d92 100644 --- a/Makefile +++ b/Makefile @@ -68,6 +68,7 @@ CCAN_OBJS := \ ccan-isaac64.o \ ccan-list.o \ ccan-mem.o \ + ccan-membuf.o \ ccan-noerr.o \ ccan-opt-helpers.o \ ccan-opt-parse.o \ @@ -129,6 +130,7 @@ CCAN_HEADERS := \ $(CCANDIR)/ccan/likely/likely.h \ $(CCANDIR)/ccan/list/list.h \ $(CCANDIR)/ccan/mem/mem.h \ + $(CCANDIR)/ccan/membuf/membuf.h \ $(CCANDIR)/ccan/noerr/noerr.h \ $(CCANDIR)/ccan/opt/opt.h \ $(CCANDIR)/ccan/opt/private.h \ @@ -601,3 +603,5 @@ ccan-str-base32.o: $(CCANDIR)/ccan/str/base32/base32.c $(CC) $(CFLAGS) -c -o $@ $< ccan-utf8.o: $(CCANDIR)/ccan/utf8/utf8.c $(CC) $(CFLAGS) -c -o $@ $< +ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c + $(CC) $(CFLAGS) -c -o $@ $< diff --git a/ccan/README b/ccan/README index 86d3a9d11..63b5a00e6 100644 --- a/ccan/README +++ b/ccan/README @@ -1,3 +1,3 @@ CCAN imported from http://ccodearchive.net. -CCAN version: init-2442-ga8722345 +CCAN version: init-2446-g1b4ed377 diff --git a/ccan/ccan/crypto/shachain/tools/Makefile b/ccan/ccan/crypto/shachain/tools/Makefile index a1b0177c7..f14646ff0 100644 --- a/ccan/ccan/crypto/shachain/tools/Makefile +++ b/ccan/ccan/crypto/shachain/tools/Makefile @@ -5,7 +5,7 @@ CFLAGS=-Wall -Werror -O3 -I$(CCANDIR) -DSHACHAIN_BITS=48 #CFLAGS=-Wall -Werror -g3 -I$(CCANDIR) -DSHACHAIN_BITS=48 # 48 bit index for shachain. This is what lightning uses. -CCAN_OBJS:=ccan-str.o ccan-err.o ccan-hex.o ccan-shachain.o ccan-sha256.o ccan-rbuf.o +CCAN_OBJS:=ccan-str.o ccan-err.o ccan-hex.o ccan-shachain.o ccan-sha256.o ccan-rbuf.o ccan-membuf.o all: shachain48 @@ -15,6 +15,7 @@ shachain48.o: $(CCANDIR)/ccan/crypto/shachain/shachain.h \ $(CCANDIR)/ccan/str/hex/hex.h \ $(CCANDIR)/ccan/str/str.h \ $(CCANDIR)/ccan/err/err.h \ + $(CCANDIR)/ccan/membuf/membuf.h \ $(CCANDIR)/ccan/rbuf/rbuf.h shachain48.o $(CCAN_OBJS): $(CCANDIR)/config.h @@ -37,3 +38,5 @@ ccan-sha256.o: $(CCANDIR)/ccan/crypto/sha256/sha256.c $(CC) $(CFLAGS) -c -o $@ $< ccan-rbuf.o: $(CCANDIR)/ccan/rbuf/rbuf.c $(CC) $(CFLAGS) -c -o $@ $< +ccan-membuf.o: $(CCANDIR)/ccan/membuf/membuf.c + $(CC) $(CFLAGS) -c -o $@ $< diff --git a/ccan/ccan/crypto/shachain/tools/shachain48.c b/ccan/ccan/crypto/shachain/tools/shachain48.c index 5cc910909..85dbf062d 100644 --- a/ccan/ccan/crypto/shachain/tools/shachain48.c +++ b/ccan/ccan/crypto/shachain/tools/shachain48.c @@ -16,9 +16,9 @@ int main(int argc, char *argv[]) char *p; shachain_init(&s); - rbuf_init(&rbuf, STDIN_FILENO, malloc(size), size); + rbuf_init(&rbuf, STDIN_FILENO, malloc(size), size, membuf_realloc); - while ((p = rbuf_read_str(&rbuf, '\n', realloc)) != NULL) { + while ((p = rbuf_read_str(&rbuf, '\n')) != NULL) { struct sha256 hash; unsigned long long idx; diff --git a/ccan/ccan/membuf/LICENSE b/ccan/ccan/membuf/LICENSE new file mode 120000 index 000000000..2354d1294 --- /dev/null +++ b/ccan/ccan/membuf/LICENSE @@ -0,0 +1 @@ +../../licenses/BSD-MIT \ No newline at end of file diff --git a/ccan/ccan/membuf/_info b/ccan/ccan/membuf/_info new file mode 100644 index 000000000..bdcbce2b2 --- /dev/null +++ b/ccan/ccan/membuf/_info @@ -0,0 +1,51 @@ +#include "config.h" +#include +#include + +/** + * membuf - simple linear memory buffer routines. + * + * It's common to want a linear memory buffer, where you can get memory on + * the end, and consume memory from the start. The details of actually + * when to enlarge or move the buffer are slightly nontrivial, so they're + * encapsulated here. + * + * License: BSD-MIT + * Author: Rusty Russell + * + * Example: + * #include + * #include + * #include + * + * // Given "hello world" outputs helloworld + * // Given "hello there world" outputs hellothereworld + * int main(int argc, char *argv[]) + * { + * MEMBUF(char) charbuf; + * + * membuf_init(&charbuf, malloc(10), 10, membuf_realloc); + * + * for (int i = 1; i < argc; i++) + * strcpy(membuf_add(&charbuf, strlen(argv[i])), argv[i]); + * + * // This is dumb, we could do all at once, but shows technique. + * while (membuf_num_elems(&charbuf) > 0) + * printf("%c", *(char *)membuf_consume(&charbuf, 1)); + * printf("\n"); + * return 0; + * } + */ +int main(int argc, char *argv[]) +{ + /* Expect exactly one argument */ + if (argc != 2) + return 1; + + if (strcmp(argv[1], "depends") == 0) { + printf("ccan/tcon\n"); + return 0; + } + + return 1; +} diff --git a/ccan/ccan/membuf/membuf.c b/ccan/ccan/membuf/membuf.c new file mode 100644 index 000000000..39841d970 --- /dev/null +++ b/ccan/ccan/membuf/membuf.c @@ -0,0 +1,60 @@ +/* MIT (BSD) license - see LICENSE file for details */ +#include +#include +#include +#include + +void membuf_init_(struct membuf *mb, + void *elems, size_t num_elems, size_t elemsize, + void *(*expandfn)(struct membuf *, void *, size_t)) +{ + + mb->start = mb->end = 0; + mb->max_elems = num_elems; + mb->elems = elems; + mb->expandfn = expandfn; +} + +size_t membuf_prepare_space_(struct membuf *mb, + size_t num_extra, size_t elemsize) +{ + char *oldstart = membuf_elems_(mb, elemsize); + + /* Always reset in the trivial empty case. */ + if (mb->start == mb->end) + mb->start = mb->end = 0; + + if (membuf_num_space_(mb) >= num_extra) + return 0; + + /* There are two ways to make space: enlarge buffer, and memmove + * down. We use a simple heuristic: if we are using less than half + * the buffer, and memmove would get us sufficient space, do that. */ + if (membuf_num_elems_(mb) <= mb->max_elems / 2 + && membuf_num_elems_(mb) + num_extra <= mb->max_elems) { + memmove(mb->elems, oldstart, (mb->end - mb->start) * elemsize); + mb->end -= mb->start; + mb->start = 0; + } else { + void *expand; + + /* Since we're going to expand, at least double. */ + if (num_extra < mb->max_elems) + num_extra = mb->max_elems; + + expand = mb->expandfn(mb, mb->elems, + (mb->max_elems + num_extra) * elemsize); + if (!expand) { + errno = ENOMEM; + } else { + mb->max_elems += num_extra; + mb->elems = expand; + } + } + return (char *)membuf_elems_(mb, elemsize) - oldstart; +} + +void *membuf_realloc(struct membuf *mb, void *rawelems, size_t newsize) +{ + return realloc(rawelems, newsize); +} diff --git a/ccan/ccan/membuf/membuf.h b/ccan/ccan/membuf/membuf.h new file mode 100644 index 000000000..aebfca27d --- /dev/null +++ b/ccan/ccan/membuf/membuf.h @@ -0,0 +1,236 @@ +/* MIT (BSD) license - see LICENSE file for details */ +#ifndef CCAN_MEMBUF_H +#define CCAN_MEMBUF_H +#include "config.h" +#include +#include + +/** + * struct membuf - representation of a memory buffer. + * + * It's exposed here to allow you to embed it and so we can inline the + * trivial functions. + */ +struct membuf { + /* These are the cursors into buf elements */ + size_t start; + size_t end; + + /* Number of elements in buf */ + size_t max_elems; + /* The buffer; at this low-level, untyped. */ + char *elems; + + void *(*expandfn)(struct membuf *, void *elems, size_t max_elems); +}; + +/** + * MEMBUF - declare a type-specific membuf + * @membertype: type for this buffer's values. + * + * You use this to create your own typed membuf. + * + * Example: + * MEMBUF(int *) intp_membuf; + * printf("Address of our int * membuf = %p\n", &intp_membuf); + */ +#define MEMBUF(membertype) \ + TCON_WRAP(struct membuf, membertype canary) + +/** + * membuf_init - initialize a type-specfic membuf. + * @mb: the MEMBUF() declared membuf. + * @elems: the initial buffer, if any. + * @max_elems: the initial space @elems, in number of elements. + * @expandfn: the function to enlarge buf (eg. membuf_realloc). + * + * Example: + * membuf_init(&intp_membuf, NULL, 0, membuf_realloc); + */ +#define membuf_init(mb, elems, num, expandfn) \ + membuf_init_(tcon_unwrap(tcon_check_ptr((mb), canary, (elems))), \ + (elems), (num), tcon_sizeof((mb), canary), (expandfn)) + +void membuf_init_(struct membuf *mb, + void *elems, size_t max_elems, size_t elemsize, + void *(*expandfn)(struct membuf *, void *, size_t)); + +/** + * membuf_realloc - simple membuf helper to do realloc(). + * + * Assumes initial buffer was NULL, or malloc(). + */ +void *membuf_realloc(struct membuf *mb, void *rawelems, size_t newsize); + +/** + * membuf_num_elems - number of populated elements in the membuf. + * @mb: the MEMBUF() declared membuf. + */ +#define membuf_num_elems(mb) membuf_num_elems_(tcon_unwrap(mb)) + +static inline size_t membuf_num_elems_(const struct membuf *mb) +{ + return mb->end - mb->start; +} + +/** + * membuf_elems - pointer to the populated elements in the membuf. + * @mb: the MEMBUF() declared membuf. + */ +#define membuf_elems(mb) \ + tcon_cast_ptr(mb, canary, \ + membuf_elems_(tcon_unwrap(mb), tcon_sizeof((mb), canary))) + +static inline void *membuf_elems_(const struct membuf *mb, size_t elemsize) +{ + return mb->elems + mb->start * elemsize; +} + +/** + * membuf_consume - we've used up this many membuf_elems. + * @mb: the MEMBUF() declared membuf. + * @num: the number of elems. + * + * Returns a pointer to the old start of membuf, so you can mark consumed + * and actually process in a single call. + */ +#define membuf_consume(mb, num) \ + tcon_cast_ptr(mb, canary, \ + membuf_consume_(tcon_unwrap(mb), (num), \ + tcon_sizeof((mb), canary))) + +static inline void *membuf_consume_(struct membuf *mb, + size_t num, size_t elemsize) +{ + void *old_start = membuf_elems_(mb, elemsize); + assert(num <= membuf_num_elems_(mb)); + mb->start += num; + + return old_start; +} + +/** + * membuf_num_space - number of unpopulated elements at end of the membuf. + * @mb: the MEMBUF() declared membuf. + */ +#define membuf_num_space(mb) membuf_num_space_(tcon_unwrap(mb)) + +static inline size_t membuf_num_space_(const struct membuf *mb) +{ + return mb->max_elems - mb->end; +} + +/** + * membuf_space - pointer to the unpopulated elements at end of membuf. + * @mb: the MEMBUF() declared membuf. + */ +#define membuf_space(mb) \ + tcon_cast_ptr(mb, canary, \ + membuf_space_(tcon_unwrap(mb), tcon_sizeof((mb), canary))) + +static inline void *membuf_space_(struct membuf *mb, size_t elemsize) +{ + return mb->elems + mb->end * elemsize; +} + +/** + * membuf_added - declare that we've added this many elements. + * @mb: the MEMBUF() declared membuf. + * @n: the number of elements we added (must be < membuf_num_space()). + */ +#define membuf_added(mb, num) \ + membuf_added_(tcon_unwrap(mb), (num)) + +static inline void membuf_added_(struct membuf *mb, size_t num) +{ + assert(num <= membuf_num_space_(mb)); + mb->end += num; +} + +/** + * membuf_prepare_space - internal routine to make sure we've got space. + * @mb: the MEMBUF() declared membuf. + * @num_extra: the minimum number of elements of space we need + * + * Usually you wouldn't call this yourself; see membuf_add() below. But + * you might use this if you need to know about moves within mb->elements + * so you can adjust your own pointers/offsets. + * + * It returns the offset *in bytes* between the old locations and the new. + * This is because it may not be a whole number of elements, in the case + * of realloc! + * + * If you want to check for expandfn failure (which sets errno to + * ENOMEM), you can check if membuf_num_space() is < num_extra which will + * never otherwise happen. + */ +#define membuf_prepare_space(mb, num_extra) \ + membuf_prepare_space_(tcon_unwrap(mb), \ + (num_extra), \ + tcon_sizeof((mb), canary)) + +size_t membuf_prepare_space_(struct membuf *mb, + size_t num_extra, size_t elemsize); + +/** + * membuf_add - add to the end of the membuf. + * @mb: the MEMBUF() declared membuf. + * @num: the number of elements (must be that much space available!). + * + * Returns the pointer to the space just added, in case you want to + * populate it afterwards. + * + * Note that this may invalidate existing buf pointers! If you want to + * avoid that, call membuf_prepare_space(mb, num) first. + */ +#define membuf_add(mb, num) \ + tcon_cast_ptr(mb, canary, \ + membuf_add_(tcon_unwrap(mb), (num), \ + tcon_sizeof((mb), canary))) + +static inline void *membuf_add_(struct membuf *mb, size_t num, size_t elemsize) +{ + void *oldend; + membuf_prepare_space_(mb, num, elemsize); + + oldend = membuf_space_(mb, elemsize); + /* We assume expandfn succeeded. */ + membuf_added_(mb, num); + + return oldend; +} + +/** + * membuf_unadd - remove this many added elements. + * @mb: the MEMBUF() declared membuf. + * @n: the number of elements we want to "unadd" (must be < membuf_num_elems()). + */ +#define membuf_unadd(mb, num) \ + membuf_unadd_(tcon_unwrap(mb), (num)) + +static inline void membuf_unadd_(struct membuf *mb, size_t num) +{ + assert(num <= membuf_num_elems_(mb)); + mb->end -= num; +} + +/** + * membuf_cleanup - reset membuf, return elems array for freeing. + * @mb: the MEMBUF() declared membuf. + * + * The mb will be empty after this, and crash if you try to expand it. + * You can membuf_init() it again, however. + * + * Example: + * free(membuf_cleanup(&intp_membuf)); + */ +#define membuf_cleanup(mb) membuf_cleanup_(tcon_unwrap(mb)) + +static inline void *membuf_cleanup_(struct membuf *mb) +{ + mb->start = mb->end = mb->max_elems = 0; + mb->expandfn = NULL; + + return mb->elems; +} +#endif /* CCAN_MEMBUF_H */ diff --git a/ccan/ccan/membuf/test/run.c b/ccan/ccan/membuf/test/run.c new file mode 100644 index 000000000..4ecc24af8 --- /dev/null +++ b/ccan/ccan/membuf/test/run.c @@ -0,0 +1,103 @@ +#include +#include +#include + +static int num_realloc, num_memmove; + +void *memmove_test(void *dest, const void *src, size_t n); +void *realloc_test(void *ptr, size_t size); + +void *memmove_test(void *dest, const void *src, size_t n) +{ + num_memmove++; + return memmove(dest, src, n); +} + +void *realloc_test(void *ptr, size_t size) +{ + num_realloc++; + return realloc(ptr, size); +} + +#undef memmove +#define memmove memmove_test + +#undef realloc +#define realloc realloc_test + +/* Include the C files directly. */ +#include +#include + +int main(void) +{ + int prev_reallocs; + MEMBUF(int) intbuf; + + /* This is how many tests you plan to run */ + plan_tests(13 + 100 * 4 + 999); + + membuf_init(&intbuf, malloc(10 * sizeof(int)), 10, membuf_realloc); + ok1(membuf_num_elems(&intbuf) == 0); + ok1(membuf_num_space(&intbuf) == 10); + ok1(membuf_space(&intbuf) != NULL); + + /* Add 100 ints. */ + for (int i = 0; i < 100; i++) { + memcpy(membuf_add(&intbuf, 1), &i, sizeof(i)); + ok1(membuf_num_elems(&intbuf) == i+1); + + /* Make sure membuf_elems works */ + if (i == 0) + ok1(memcmp(membuf_elems(&intbuf), &i, sizeof(i)) == 0); + } + + + /* Pull 100 ints. */ + for (int i = 0; i < 100; i++) { + ok1(memcmp(membuf_consume(&intbuf, 1), &i, sizeof(i)) == 0); + ok1(membuf_num_elems(&intbuf) == 100 - i - 1); + } + + /* Should not have continuously realloced or memmoved */ + ok1(num_realloc < 10); + ok1(num_memmove == 0); + + /* Doing it again should give 0 reallocs. */ + prev_reallocs = num_realloc; + for (int i = 0; i < 100; i++) { + memcpy(membuf_add(&intbuf, 1), &i, sizeof(i)); + ok1(membuf_num_elems(&intbuf) == i+1); + } + ok1(num_realloc == prev_reallocs); + ok1(num_memmove == 0); + + membuf_consume(&intbuf, 100); + + /* Keep a single element in the queue, make sure we don't realloc! */ + for (int i = 0; i < 1000; i++) { + memcpy(membuf_add(&intbuf, 1), &i, sizeof(i)); + if (i > 0) { + int prev = i - 1; + ok1(memcmp(membuf_consume(&intbuf, 1), + &prev, sizeof(prev)) == 0); + } + } + + ok1(num_realloc == prev_reallocs); + /* Should have moved occasionally. */ + ok1(num_memmove < 20); + + ok1(membuf_consume(&intbuf, 1)); + ok1(membuf_num_elems(&intbuf) == 0); + + /* Force it to more-than-double; make sure that works! */ + memset(membuf_add(&intbuf, 300), 0, 300*sizeof(int)); + ok1(membuf_num_elems(&intbuf) == 300); + + /* Free buffer so valgrind is happy. */ + free(membuf_cleanup(&intbuf)); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/ccan/rbuf/_info b/ccan/ccan/rbuf/_info index 7c9224d03..8a7d46e8c 100644 --- a/ccan/ccan/rbuf/_info +++ b/ccan/ccan/rbuf/_info @@ -23,19 +23,19 @@ * char *word; * * if (argv[1]) { - * if (!rbuf_open(&in, argv[1], NULL, 0)) + * if (!rbuf_open(&in, argv[1], NULL, 0, membuf_realloc)) * err(1, "Failed opening %s", argv[1]); * } else - * rbuf_init(&in, STDIN_FILENO, NULL, 0); + * rbuf_init(&in, STDIN_FILENO, NULL, 0, membuf_realloc); * - * while ((word = rbuf_read_str(&in, ' ', realloc)) != NULL) + * while ((word = rbuf_read_str(&in, ' ')) != NULL) * printf("%s*", word); * * if (errno) * err(1, "Reading %s", argv[1] ? argv[1] : ""); * * // Free the buffer, just because we can. - * free(in.buf); + * free(rbuf_cleanup(&in)); * return 0; * } */ @@ -46,6 +46,7 @@ int main(int argc, char *argv[]) return 1; if (strcmp(argv[1], "depends") == 0) { + printf("ccan/membuf\n"); return 0; } diff --git a/ccan/ccan/rbuf/rbuf.c b/ccan/ccan/rbuf/rbuf.c index 0e210a9e3..d8d658d37 100644 --- a/ccan/ccan/rbuf/rbuf.c +++ b/ccan/ccan/rbuf/rbuf.c @@ -7,21 +7,17 @@ #include #include -bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max) +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)) { int fd = open(name, O_RDONLY); if (fd >= 0) { - rbuf_init(rbuf, fd, buf, buf_max); + rbuf_init(rbuf, fd, buf, buf_max, expandfn); return true; } return false; } -static size_t rem(const struct rbuf *buf) -{ - return buf->buf_end - (buf->start + buf->len); -} - size_t rbuf_good_size(int fd) { struct stat st; @@ -31,100 +27,78 @@ size_t rbuf_good_size(int fd) return 4096; } -static bool enlarge_buf(struct rbuf *buf, size_t len, - void *(*resize)(void *buf, size_t len)) +static ssize_t get_more(struct rbuf *rbuf) { - char *new; - if (!resize) { - errno = ENOMEM; - return false; - } - if (!len) - len = rbuf_good_size(buf->fd); - new = resize(buf->buf, len); - if (!new) - return false; - buf->start += (new - buf->buf); - buf->buf = new; - buf->buf_end = new + len; - return true; -} + ssize_t r; -static ssize_t get_more(struct rbuf *rbuf, - void *(*resize)(void *buf, size_t len)) -{ - size_t r; + /* This is so we only call rbuf_good_size once. */ + if (tcon_unwrap(&rbuf->m)->max_elems == 0) + membuf_prepare_space(&rbuf->m, rbuf_good_size(rbuf->fd)); + else /* membuf doubles internally, so just ask for anything. */ + membuf_prepare_space(&rbuf->m, 1); - if (rbuf->start + rbuf->len == rbuf->buf_end) { - if (!enlarge_buf(rbuf, (rbuf->buf_end - rbuf->buf) * 2, resize)) - return -1; - } + /* This happens if realloc fails (errno already ENOMEM) */ + if (!membuf_num_space(&rbuf->m)) + return -1; - r = read(rbuf->fd, rbuf->start + rbuf->len, rem(rbuf)); + r = read(rbuf->fd, membuf_space(&rbuf->m), membuf_num_space(&rbuf->m)); if (r <= 0) return r; - rbuf->len += r; + membuf_add(&rbuf->m, r); return r; } -void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +void *rbuf_fill_all(struct rbuf *rbuf) { ssize_t r; - /* Move back to start of buffer if we're empty. */ - if (!rbuf->len) - rbuf->start = rbuf->buf; - - while ((r = get_more(rbuf, resize)) != 0) + while ((r = get_more(rbuf)) != 0) if (r < 0) return NULL; - return rbuf->start; + return rbuf_start(rbuf); } -void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)) +void *rbuf_fill(struct rbuf *rbuf) { - if (!rbuf->len) { - rbuf->start = rbuf->buf; - if (get_more(rbuf, resize) < 0) + if (!rbuf_len(rbuf)) { + if (get_more(rbuf) < 0) return NULL; } - return rbuf->start; + return rbuf_start(rbuf); } -char *rbuf_read_str(struct rbuf *rbuf, char term, - void *(*resize)(void *buf, size_t len)) +char *rbuf_read_str(struct rbuf *rbuf, char term) { - char *p, *ret; + char *p; ssize_t r = 0; size_t prev = 0; - /* Move back to start of buffer if we're empty. */ - if (!rbuf->len) - rbuf->start = rbuf->buf; - - while (!(p = memchr(rbuf->start + prev, term, rbuf->len - prev))) { + while (!(p = memchr(membuf_elems(&rbuf->m) + prev, + term, + membuf_num_elems(&rbuf->m) - prev))) { prev += r; - r = get_more(rbuf, resize); + r = get_more(rbuf); if (r < 0) return NULL; /* EOF with no term. */ if (r == 0) { + char *ret; + size_t len = rbuf_len(rbuf); + /* Nothing read at all? */ - if (!rbuf->len && term) { + if (!len && term) { errno = 0; return NULL; } + /* Put term after input (get_more made room). */ - assert(rbuf->start + rbuf->len < rbuf->buf_end); - rbuf->start[rbuf->len] = '\0'; - ret = rbuf->start; - rbuf_consume(rbuf, rbuf->len); + assert(membuf_num_space(&rbuf->m) > 0); + ret = membuf_consume(&rbuf->m, len); + ret[len] = '\0'; return ret; } } *p = '\0'; - ret = rbuf->start; - rbuf_consume(rbuf, p + 1 - ret); - return ret; + return membuf_consume(&rbuf->m, p + 1 - (char *)rbuf_start(rbuf)); } diff --git a/ccan/ccan/rbuf/rbuf.h b/ccan/ccan/rbuf/rbuf.h index ab1504a7c..7b5306c4a 100644 --- a/ccan/ccan/rbuf/rbuf.h +++ b/ccan/ccan/rbuf/rbuf.h @@ -5,41 +5,36 @@ #include // For UCHAR_MAX #include #include +#include struct rbuf { int fd; - - /* Where to read next. */ - char *start; - /* How much of what is there is valid. */ - size_t len; - - /* The entire buffer memory we have to work with. */ - char *buf, *buf_end; + MEMBUF(char) m; }; /** * rbuf_init - set up a buffer. - * @buf: the struct rbuf. + * @rbuf: the struct rbuf. * @fd: the file descriptor. * @buf: the buffer to use. * @buf_max: the size of the buffer. + * @expandfn: usually membuf_realloc. */ -static inline void rbuf_init(struct rbuf *buf, - int fd, char *buffer, size_t buf_max) +static inline void rbuf_init(struct rbuf *rbuf, + int fd, char *buffer, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)) { - buf->fd = fd; - buf->start = buf->buf = buffer; - buf->len = 0; - buf->buf_end = buffer + buf_max; + rbuf->fd = fd; + membuf_init(&rbuf->m, buffer, buf_max, expandfn); } /** * rbuf_open - set up a buffer by opening a file. - * @buf: the struct rbuf. + * @rbuf: the struct rbuf. * @filename: the filename * @buf: the buffer to use. * @buf_max: the size of the buffer. + * @expandfn: usually membuf_realloc. * * Returns false if the open fails. If @buf_max is 0, then the buffer * will be resized to rbuf_good_size() on first rbuf_fill. @@ -47,10 +42,11 @@ static inline void rbuf_init(struct rbuf *buf, * Example: * struct rbuf in; * - * if (!rbuf_open(&in, "foo", NULL, 0)) + * if (!rbuf_open(&in, "foo", NULL, 0, membuf_realloc)) * err(1, "Could not open foo"); */ -bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max); +bool rbuf_open(struct rbuf *rbuf, const char *name, char *buf, size_t buf_max, + void *(*expandfn)(struct membuf *, void *, size_t)); /** * rbuf_good_size - get a good buffer size for this fd. @@ -62,73 +58,82 @@ size_t rbuf_good_size(int fd); /** * rbuf_fill - read into a buffer if it's empty. - * @buf: the struct rbuf - * @resize: the call to resize the buffer. + * @rbuf: the struct rbuf * - * If @resize is needed and is NULL, or returns false, rbuf_read will - * return NULL (with errno set to ENOMEM). If a read fails, then NULL - * is also returned. If there is nothing more to read, it will return - * NULL with errno set to 0. Otherwise, returns @buf->start; @buf->len - * is the valid length of the buffer. + * If @expandfn fails, rbuf_fill will return NULL (with errno set to ENOMEM). + * If a read fails, then NULL is also returned. If there is nothing more to + * read, it will return NULL with errno set to 0. Otherwise, returns first + * populated bytes (aka. rbuf_start()); rbuf_len() is the valid length of the + * buffer. * * You need to call rbuf_consume() to mark data in the buffer as * consumed. * * Example: - * while (rbuf_fill(&in, realloc)) { - * printf("%.*s\n", (int)in.len, in.start); - * rbuf_consume(&in, in.len); + * while (rbuf_fill(&in)) { + * printf("%.*s\n", (int)rbuf_len(&in), rbuf_start(&in)); + * rbuf_consume(&in, rbuf_len(&in)); * } * if (errno) * err(1, "reading foo"); */ -void *rbuf_fill(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); +void *rbuf_fill(struct rbuf *rbuf); /** * rbuf_consume - helper to use up data in a buffer. - * @buf: the struct rbuf + * @rbuf: the struct rbuf * @len: the length (from @buf->start) you used. * * After rbuf_fill() you should indicate the data you've used with * rbuf_consume(). That way rbuf_fill() will know if it has anything * to do. */ -static inline void rbuf_consume(struct rbuf *buf, size_t len) +static inline void rbuf_consume(struct rbuf *rbuf, size_t len) +{ + membuf_consume(&rbuf->m, len); +} + +/** + * rbuf_len - helper to determine how many bytes in rbuf + * @rbuf: the struct rbuf + */ +static inline size_t rbuf_len(const struct rbuf *rbuf) +{ + return membuf_num_elems(&rbuf->m); +} + +/** + * rbuf_start - helper to get pointert to unconsumed bytes in rbuf + * @rbuf: the struct rbuf + */ +static inline char *rbuf_start(const struct rbuf *rbuf) { - buf->len -= len; - buf->start += len; + return membuf_elems(&rbuf->m); } /** * rbuf_fill_all - read rest of file into a buffer. - * @buf: the struct rbuf - * @resize: the call to resize the buffer. + * @rbuf: the struct rbuf * - * If @resize is needed and is NULL, or returns false, rbuf_read_all - * will return NULL (with errno set to ENOMEM). If a read fails, - * then NULL is also returned, otherwise returns @buf->start. + * If a read or @expandfn fails then NULL returned, otherwise returns + * @rbuf->start. * * Example: - * if (!rbuf_fill_all(&in, realloc)) { - * if (errno) - * err(1, "reading foo"); - * } + * if (!rbuf_fill_all(&in)) + * err(1, "reading foo"); */ -void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); +void *rbuf_fill_all(struct rbuf *rbuf); /** * rbuf_read_str - fill into a buffer up to a terminator, and consume string. - * @buf: the struct rbuf + * @rbuf: the struct rbuf * @term: the character to terminate the read. - * @resize: the call to resize the buffer. * - * If @resize is needed and is NULL, or returns false, rbuf_read_str - * will return NULL (with errno set to ENOMEM). If a read fails, - * then NULL is also returned, otherwise the next string. It - * replaces the terminator @term (if any) with NUL, otherwise NUL + * If a read or @expandfn fails, then NULL is returned, otherwise the next + * string. It replaces the terminator @term (if any) with NUL, otherwise NUL * is placed after EOF. If you need to, you can tell this has happened - * because the nul terminator will be at @buf->start (normally it will - * be at @buf->start - 1). + * because the nul terminator will be at rbuf_start(@rbuf) (normally it will be + * at rbuf_start(@rbuf) - 1). * * If there is nothing remaining to be read, NULL is returned with * errno set to 0, unless @term is NUL, in which case it returns the @@ -140,7 +145,7 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); * Example: * char *line; * - * line = rbuf_read_str(&in, '\n', realloc); + * line = rbuf_read_str(&in, '\n'); * if (!line) { * if (errno) * err(1, "reading foo"); @@ -150,7 +155,20 @@ void *rbuf_fill_all(struct rbuf *rbuf, void *(*resize)(void *buf, size_t len)); * printf("First line is %s\n", line); * */ -char *rbuf_read_str(struct rbuf *rbuf, char term, - void *(*resize)(void *buf, size_t len)); +char *rbuf_read_str(struct rbuf *rbuf, char term); +/** + * rbuf_cleanup - reset rbuf, return buffer for freeing. + * @rbuf: the struct rbuf + * + * The rbuf will be empty after this, and crash if you try to use it. + * You can rbuf_init() it again, however. + * + * Example: + * free(rbuf_cleanup(&in)); + */ +static inline char *rbuf_cleanup(struct rbuf *rbuf) +{ + return membuf_cleanup(&rbuf->m); +} #endif /* CCAN_RBUF_H */ diff --git a/ccan/ccan/rbuf/test/run-all.c b/ccan/ccan/rbuf/test/run-all.c index 2cd0b4241..c41ee20dd 100644 --- a/ccan/ccan/rbuf/test/run-all.c +++ b/ccan/ccan/rbuf/test/run-all.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -24,25 +32,29 @@ int main(void) } close(fd); - ok1(rbuf_open(&in, "run-all-file", NULL, 0)); - /* Can't fill without realloc. */ - ok1(!rbuf_fill(&in, NULL)); + ok1(rbuf_open(&in, "run-all-file", NULL, 0, test_realloc)); + /* Can't fill if realloc fails. */ + test_realloc_fail = true; + ok1(!rbuf_fill(&in)); ok1(errno == ENOMEM); - ok1(rbuf_fill(&in, realloc)); + test_realloc_fail = false; + ok1(rbuf_fill(&in)); /* But can't load in whole file. */ - ok1(!rbuf_fill_all(&in, NULL)); + test_realloc_fail = true; + ok1(!rbuf_fill_all(&in)); ok1(errno == ENOMEM); - ok1(rbuf_fill_all(&in, realloc)); - ok1(in.len == size); + test_realloc_fail = false; + ok1(rbuf_fill_all(&in)); + ok1(rbuf_len(&in) == size); for (i = 0; i * sizeof(buf) < size; i++) { memset(buf, 0x42 + i, sizeof(buf)); - if (memcmp(buf, in.start, sizeof(buf)) != 0) { + if (memcmp(buf, rbuf_start(&in), sizeof(buf)) != 0) { fail("Bad buffer contents"); break; } rbuf_consume(&in, sizeof(buf)); } - free(in.buf); + free(rbuf_cleanup(&in)); /* This exits depending on whether all tests passed */ return exit_status(); diff --git a/ccan/ccan/rbuf/test/run-open.c b/ccan/ccan/rbuf/test/run-open.c index 5678e42e9..4e8277f22 100644 --- a/ccan/ccan/rbuf/test/run-open.c +++ b/ccan/ccan/rbuf/test/run-open.c @@ -14,9 +14,9 @@ int main(void) /* This is how many tests you plan to run */ plan_tests(5); - ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0)); + ok1(!rbuf_open(&in, "nonexistent-file", NULL, 0, NULL)); ok1(errno == ENOENT); - ok1(rbuf_open(&in, "test/run-open.c", NULL, 0)); + ok1(rbuf_open(&in, "test/run-open.c", NULL, 0, NULL)); ok1(close(in.fd) == 0); /* If this fails to stat, it should fall back */ ok1(rbuf_good_size(in.fd) == 4096); diff --git a/ccan/ccan/rbuf/test/run-partial-read.c b/ccan/ccan/rbuf/test/run-partial-read.c index 362e6b601..7ecf79d35 100644 --- a/ccan/ccan/rbuf/test/run-partial-read.c +++ b/ccan/ccan/rbuf/test/run-partial-read.c @@ -27,7 +27,7 @@ int main(void) int i, fd = open("test/run.c", O_RDONLY); /* This is how many tests you plan to run */ - plan_tests(140); + plan_tests(160); /* Grab ourselves for comparison. */ buf[full_read(fd, buf, sizeof(buf))] = '\0'; @@ -41,26 +41,26 @@ int main(void) } lines[i] = NULL; - rbuf_init(&in, fd, malloc(31), 31); + rbuf_init(&in, fd, malloc(31), 31, membuf_realloc); ok1(in.fd == fd); - ok1(in.buf_end - in.buf == 31); - p = rbuf_read_str(&in, '\n', NULL); + ok1(membuf_num_space(&in.m) == 31); + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[0]) == 0); - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[1]) == 0); for (i = 2; lines[i]; i++) { - ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(p = rbuf_read_str(&in, '\n')); ok1(strcmp(p, lines[i]) == 0); } - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(errno == 0); ok1(p == NULL); - free(in.buf); + free(rbuf_cleanup(&in)); /* This exits depending on whether all tests passed */ return exit_status(); diff --git a/ccan/ccan/rbuf/test/run-term-eof.c b/ccan/ccan/rbuf/test/run-term-eof.c index 097dcbbda..86d7c18d4 100644 --- a/ccan/ccan/rbuf/test/run-term-eof.c +++ b/ccan/ccan/rbuf/test/run-term-eof.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -14,7 +22,7 @@ int main(void) int fd = open("test/run-term-eof.c", O_RDONLY), len; /* This is how many tests you plan to run */ - plan_tests(6); + plan_tests(10); /* Grab ourselves for comparison. */ len = read(fd, buf, sizeof(buf)); @@ -22,23 +30,35 @@ int main(void) lseek(fd, SEEK_SET, 0); /* We have exact-size buffer, which causes problems adding term. */ - rbuf_init(&in, fd, malloc(len), len); - p = rbuf_read_str(&in, 64, NULL); /* At symbol does not appear. */ + rbuf_init(&in, fd, malloc(len), len, test_realloc); + test_realloc_fail = true; + p = rbuf_read_str(&in, 64); /* At symbol does not appear. */ ok1(errno == ENOMEM); ok1(!p); /* This should succeed... */ - p = rbuf_read_str(&in, 64, realloc); + test_realloc_fail = false; + p = rbuf_read_str(&in, 64); ok1(p); ok1(strcmp(p, buf) == 0); - free(in.buf); + ok1(rbuf_start(&in) == p + strlen(p)); + free(rbuf_cleanup(&in)); /* Try again. */ lseek(fd, SEEK_SET, 0); - rbuf_init(&in, fd, malloc(len), len); - p = rbuf_read_str(&in, 64, realloc); + rbuf_init(&in, fd, malloc(len), len, test_realloc); + p = rbuf_read_str(&in, 64); ok1(p); ok1(strcmp(p, buf) == 0); - free(in.buf); + ok1(rbuf_start(&in) == p + strlen(p)); + free(rbuf_cleanup(&in)); + + /* Normal case, we get rbuf_start after nul */ + lseek(fd, SEEK_SET, 0); + rbuf_init(&in, fd, NULL, 0, test_realloc); + p = rbuf_read_str(&in, '^'); + ok1(p); + ok1(rbuf_start(&in) == p + strlen(p) + 1); + free(rbuf_cleanup(&in)); return exit_status(); } diff --git a/ccan/ccan/rbuf/test/run.c b/ccan/ccan/rbuf/test/run.c index e57618818..593ef2708 100644 --- a/ccan/ccan/rbuf/test/run.c +++ b/ccan/ccan/rbuf/test/run.c @@ -7,6 +7,14 @@ #include #include +static bool test_realloc_fail; +static void *test_realloc(struct membuf *mb, void *buf, size_t n) +{ + if (test_realloc_fail) + return NULL; + return realloc(buf, n); +} + int main(void) { struct rbuf in; @@ -15,7 +23,7 @@ int main(void) int i, fd = open("test/run.c", O_RDONLY), len; /* This is how many tests you plan to run */ - plan_tests(144); + plan_tests(164); /* Grab ourselves for comparison. */ len = read(fd, buf, sizeof(buf)); @@ -30,39 +38,41 @@ int main(void) } lines[i] = NULL; - rbuf_init(&in, fd, malloc(31), 31); + rbuf_init(&in, fd, malloc(31), 31, test_realloc); ok1(in.fd == fd); - ok1(in.buf_end - in.buf == 31); - p = rbuf_read_str(&in, '\n', NULL); + ok1(membuf_num_space(&in.m) == 31); + test_realloc_fail = true; + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[0]) == 0); - p = rbuf_read_str(&in, '\n', realloc); + test_realloc_fail = false; + p = rbuf_read_str(&in, '\n'); ok1(p); ok1(strcmp(p, lines[1]) == 0); for (i = 2; lines[i]; i++) { - ok1(p = rbuf_read_str(&in, '\n', realloc)); + ok1(p = rbuf_read_str(&in, '\n')); ok1(strcmp(p, lines[i]) == 0); } - p = rbuf_read_str(&in, '\n', realloc); + p = rbuf_read_str(&in, '\n'); ok1(errno == 0); ok1(p == NULL); - free(in.buf); + free(rbuf_cleanup(&in)); /* Another way of reading the entire (text) file. */ lseek(fd, SEEK_SET, 0); - rbuf_init(&in, fd, NULL, 0); - p = rbuf_read_str(&in, 0, realloc); + rbuf_init(&in, fd, NULL, 0, test_realloc); + p = rbuf_read_str(&in, 0); ok1(p); ok1(strlen(p) == len); close(fd); - p = rbuf_read_str(&in, 0, realloc); + p = rbuf_read_str(&in, 0); ok1(errno == EBADF); ok1(!p); - free(in.buf); + free(rbuf_cleanup(&in)); return exit_status(); } diff --git a/connectd/tor_autoservice.c b/connectd/tor_autoservice.c index 381ab13c4..e7cb64451 100644 --- a/connectd/tor_autoservice.c +++ b/connectd/tor_autoservice.c @@ -26,7 +26,7 @@ #define MAX_TOR_ONION_V2_ADDR_LEN 16 #define MAX_TOR_ONION_V3_ADDR_LEN 56 -static void *buf_resize(void *buf, size_t len) +static void *buf_resize(struct membuf *mb, void *buf, size_t len) { tal_resize(&buf, len); return buf; @@ -49,7 +49,7 @@ static char *tor_response_line(struct rbuf *rbuf) { char *line; - while ((line = rbuf_read_str(rbuf, '\n', buf_resize)) != NULL) { + while ((line = rbuf_read_str(rbuf, '\n')) != NULL) { status_io(LOG_IO_IN, "torcontrol", line, strlen(line)); /* Weird response */ @@ -222,7 +222,7 @@ struct wireaddr *tor_autoservice(const tal_t *ctx, err(1, "Connecting stream socket to Tor service"); buffer = tal_arr(tmpctx, char, rbuf_good_size(fd)); - rbuf_init(&rbuf, fd, buffer, tal_count(buffer)); + rbuf_init(&rbuf, fd, buffer, tal_count(buffer), buf_resize); negotiate_auth(&rbuf, tor_password); onion = make_onion(ctx, &rbuf, laddr);