wire: generate marshal/unmarshal from spec.

Including tests! Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
8 years ago · 36e018161f
10 changed files with 1158 additions and 0 deletions
--- a/1
+++ b/1
@ -201,6 +201,7 @@ protocol-diagrams: $(patsubst %.script, doc/protocol-%.svg, $(notdir $(wildcard
 check: test-protocol bitcoin-tests

 include bitcoin/Makefile
+include wire/Makefile

 # Keep includes in alpha order.
 check-src-include-order/%: %
--- a/wire/Makefile
+++ b/wire/Makefile
@ -0,0 +1,33 @@
+#! /usr/bin/make
+
+# Designed to be run one level up
+wire-wrongdir:
+	$(MAKE) -C .. wire-all
+
+WIRE_OBJS := wire/gen_wire.o	\
+	wire/fromwire.o		\
+	wire/towire.o
+
+WIRE_HEADERS := wire/wire.h
+WIRE_GEN_HEADERS := wire/gen_wire.h
+WIRE_GEN_SRC := wire/gen_wire.c
+
+# They may not have the bolts.
+BOLT_EXTRACT=$(BOLTDIR)/tools/extract-formats.py
+wire/gen_wire_csv: FORCE
+	@set -e; if [ -f $(BOLT_EXTRACT) ]; then for f in $(BOLTDIR)/*.md $(BOLT_EXTRACT); do if [ $$f -nt $@ -o ! -f $@ ]; then $(BOLT_EXTRACT) --message-fields --message-types --check-alignment $(BOLTDIR)/*.md > $@; break; fi; done; fi
+
+wire/gen_wire.h: wire/tools/generate-wire.py wire/gen_wire_csv
+	wire/tools/generate-wire.py --header < wire/gen_wire_csv > $@
+
+wire/gen_wire.c: wire/tools/generate-wire.py wire/gen_wire_csv
+	wire/tools/generate-wire.py < wire/gen_wire_csv > $@
+
+wire/gen_wire.o: wire/gen_wire.h
+
+clean: wire-clean
+
+wire-clean:
+	$(RM) $(WIRE_OBJS) $(WIRE_GEN_SRC) $(WIRE_GEN_HEADERS)
+
+include wire/test/Makefile 
--- a/wire/fromwire.c
+++ b/wire/fromwire.c
@ -0,0 +1,132 @@
+#include "wire.h"
+#include <bitcoin/pubkey.h>
+#include <ccan/endian/endian.h>
+#include <ccan/mem/mem.h>
+
+/* Sets *cursor to NULL and returns NULL when extraction fails. */
+static const void *fail_pull(const u8 **cursor, size_t *max)
+{
+	*cursor = NULL;
+	*max = 0;
+	return NULL;
+}
+
+const u8 *fromwire(const u8 **cursor, size_t *max, void *copy, size_t n)
+{
+	const u8 *p = *cursor;
+
+	if (*max < n) {
+		/* Just make sure we don't leak uninitialized mem! */
+		if (copy)
+			memset(copy, 0, n);
+		return fail_pull(cursor, max);
+	}
+	*cursor += n;
+	*max -= n;
+	if (copy)
+		memcpy(copy, p, n);
+	return memcheck(p, n);
+}
+
+u8 fromwire_u8(const u8 **cursor, size_t *max)
+{
+	u8 ret;
+
+	if (!fromwire(cursor, max, &ret, sizeof(ret)))
+		return 0;
+	return ret;
+}
+
+u16 fromwire_u16(const u8 **cursor, size_t *max)
+{
+	be16 ret;
+
+	if (!fromwire(cursor, max, &ret, sizeof(ret)))
+		return 0;
+	return be16_to_cpu(ret);
+}
+	
+u32 fromwire_u32(const u8 **cursor, size_t *max)
+{
+	be32 ret;
+
+	if (!fromwire(cursor, max, &ret, sizeof(ret)))
+		return 0;
+	return be32_to_cpu(ret);
+}
+
+u64 fromwire_u64(const u8 **cursor, size_t *max)
+{
+	be64 ret;
+
+	if (!fromwire(cursor, max, &ret, sizeof(ret)))
+		return 0;
+	return be64_to_cpu(ret);
+}
+
+void fromwire_pubkey(const u8 **cursor, size_t *max, struct pubkey *pubkey)
+{
+	u8 der[PUBKEY_DER_LEN];
+
+	if (!fromwire(cursor, max, der, sizeof(der)))
+		return;
+
+	if (!pubkey_from_der(secp256k1_ctx, der, sizeof(der), pubkey))
+		fail_pull(cursor, max);
+}
+
+void fromwire_signature(const u8 **cursor, size_t *max, struct signature *sig)
+{
+	u8 compact[64];
+
+	if (!fromwire(cursor, max, compact, sizeof(compact)))
+		return;
+
+	if (secp256k1_ecdsa_signature_parse_compact(secp256k1_ctx,
+						    &sig->sig, compact)
+	    != 1)
+		fail_pull(cursor, max);
+}
+
+void fromwire_channel_id(const u8 **cursor, size_t *max,
+			 struct channel_id *channel_id)
+{
+	be32 txnum = 0;
+	u8 outnum;
+
+	channel_id->blocknum = fromwire_u32(cursor, max);
+	/* Pulling 3 bytes off wire is tricky; they're big-endian. */
+	fromwire(cursor, max, (char *)&txnum + 1, 3);
+	channel_id->txnum = be32_to_cpu(txnum);
+	fromwire(cursor, max, &outnum, 1);
+	channel_id->outnum = outnum;
+}
+	
+void fromwire_sha256(const u8 **cursor, size_t *max, struct sha256 *sha256)
+{
+	fromwire(cursor, max, sha256, sizeof(*sha256));
+}
+
+void fromwire_ipv6(const u8 **cursor, size_t *max, struct ipv6 *ipv6)
+{
+	fromwire(cursor, max, ipv6, sizeof(*ipv6));
+}
+
+void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num)
+{
+	fromwire(cursor, max, arr, num);
+}
+
+void fromwire_pad_array(const u8 **cursor, size_t *max, u8 *arr, size_t num)
+{
+	fromwire(cursor, max, arr, num);
+}
+	
+void fromwire_signature_array(const u8 **cursor, size_t *max,
+			      struct signature *arr, size_t num)
+{
+	size_t i;
+
+	for (i = 0; i < num; i++)
+		fromwire_signature(cursor, max, arr + i);
+}
--- a/wire/gen_wire_csv
+++ b/wire/gen_wire_csv
@ -0,0 +1,119 @@
+init,16
+init,0,gflen,2
+init,2,globalfeatures,gflen
+init,2+gflen,lflen,2
+init,4+gflen,localfeatures,lflen
+error,17
+error,0,channel-id,8
+error,8,len,2
+error,10,data,len
+open_channel,32
+open_channel,0,temporary-channel-id,8
+open_channel,8,funding-satoshis,8
+open_channel,16,push-msat,8
+open_channel,24,dust-limit-satoshis,8
+open_channel,32,max-htlc-value-in-flight-msat,8
+open_channel,40,channel-reserve-satoshis,8
+open_channel,48,htlc-minimum-msat,4
+open_channel,52,feerate-per-kw,4
+open_channel,56,to-self-delay,2
+open_channel,58,max-accepted-htlcs,2
+open_channel,60,funding-pubkey,33
+open_channel,93,revocation-basepoint,33
+open_channel,126,payment-basepoint,33
+open_channel,159,delayed-payment-basepoint,33
+open_channel,192,first-per-commitment-point,33
+accept_channel,33
+accept_channel,0,temporary-channel-id,8
+accept_channel,8,dust-limit-satoshis,8
+accept_channel,16,max-htlc-value-in-flight-msat,8
+accept_channel,24,channel-reserve-satoshis,8
+accept_channel,32,minimum-depth,4
+accept_channel,36,htlc-minimum-msat,4
+accept_channel,40,to-self-delay,2
+accept_channel,42,max-accepted-htlcs,2
+accept_channel,44,funding-pubkey,33
+accept_channel,77,revocation-basepoint,33
+accept_channel,110,payment-basepoint,33
+accept_channel,143,delayed-payment-basepoint,33
+accept_channel,176,first-per-commitment-point,33
+funding_created,34
+funding_created,0,temporary-channel-id,8
+funding_created,8,txid,32
+funding_created,40,output-index,1
+funding_created,41,signature,64
+funding_signed,35
+funding_signed,0,temporary-channel-id,8
+funding_signed,8,signature,64
+funding_locked,36
+funding_locked,0,temporary-channel-id,8
+funding_locked,8,channel-id,8
+funding_locked,16,announcement-node-signature,64
+funding_locked,80,announcement-bitcoin-signature,64
+funding_locked,144,next-per-commitment-point,33
+update_fee,37
+update_fee,0,channel-id,8
+update_fee,8,feerate-per-kw,4
+shutdown,38
+shutdown,0,channel-id,8
+shutdown,8,len,2
+shutdown,10,scriptpubkey,len
+closing_signed,39
+closing_signed,0,channel-id,8
+closing_signed,8,fee-satoshis,8
+closing_signed,16,signature,64
+update_add_htlc,128
+update_add_htlc,0,channel-id,8
+update_add_htlc,8,id,8
+update_add_htlc,16,amount-msat,4
+update_add_htlc,20,expiry,4
+update_add_htlc,24,payment-hash,32
+update_add_htlc,56,onion-routing-packet,1254
+update_fulfill_htlc,130
+update_fulfill_htlc,0,channel-id,8
+update_fulfill_htlc,8,id,8
+update_fulfill_htlc,16,payment-preimage,32
+update_fail_htlc,131
+update_fail_htlc,0,channel-id,8
+update_fail_htlc,8,id,8
+update_fail_htlc,16,reason,154
+commit_sig,132
+commit_sig,0,channel-id,8
+commit_sig,8,signature,64
+commit_sig,72,num-htlcs,2
+commit_sig,74,htlc-signature,num-htlcs*64
+revoke_and_ack,133
+revoke_and_ack,0,channel-id,8
+revoke_and_ack,8,per-commitment-secret,32
+revoke_and_ack,40,next-per-commitment-point,33
+revoke_and_ack,73,padding,1
+revoke_and_ack,74,num-htlc-timeouts,2
+revoke_and_ack,76,htlc-timeout-signature,num-htlc-timeouts*64
+channel_announcement,256
+channel_announcement,0,node-signature-1,64
+channel_announcement,64,node-signature-2,64
+channel_announcement,128,channel-id,8
+channel_announcement,136,bitcoin-signature-1,64
+channel_announcement,200,bitcoin-signature-2,64
+channel_announcement,264,node-id-1,33
+channel_announcement,297,node-id-2,33
+channel_announcement,330,bitcoin-key-1,33
+channel_announcement,363,bitcoin-key-2,33
+node_announcement,257
+node_announcement,0,signature,64
+node_announcement,64,timestamp,4
+node_announcement,68,ipv6,16
+node_announcement,84,port,2
+node_announcement,86,node-id,33
+node_announcement,119,rgb-color,3
+node_announcement,122,pad,2
+node_announcement,124,alias,32
+channel_update,258
+channel_update,0,signature,64
+channel_update,64,channel-id,8
+channel_update,72,timestamp,4
+channel_update,76,flags,2
+channel_update,78,expiry,2
+channel_update,80,htlc-minimum-msat,4
+channel_update,84,fee-base-msat,4
+channel_update,88,fee-proportional-millionths,4
--- a/wire/test/.gitignore
+++ b/wire/test/.gitignore
@ -0,0 +1 @@
+run-wire
--- a/wire/test/Makefile
+++ b/wire/test/Makefile
@ -0,0 +1,16 @@
+check: wire-tests
+
+# Note that these actually #include everything they need, except ccan/ and bitcoin/.
+# That allows for unit testing of statics, and special effects.
+WIRE_TEST_SRC := $(wildcard wire/test/run-*.c)
+WIRE_TEST_OBJS := $(WIRE_TEST_SRC:.c=.o)
+WIRE_TEST_PROGRAMS := $(WIRE_TEST_OBJS:.o=)
+
+update-mocks: $(WIRE_TEST_SRC:%=update-mocks/%)
+
+$(WIRE_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) libsecp256k1.a utils.o
+
+$(WIRE_TEST_OBJS): $(WIRE_HEADERS) $(BITCOIN_HEADERS) $(CORE_HEADERS) $(GEN_HEADERS) $(WIRE_GEN_HEADERS) $(WIRE_GEN_SRC) $(CCAN_HEADERS)
+
+wire-tests: $(WIRE_TEST_PROGRAMS:%=unittest/%)
+
--- a/wire/test/run-wire.c
+++ b/wire/test/run-wire.c
@ -0,0 +1,434 @@
+#include "../gen_wire.c"
+
+void towire_pad_array_orig(u8 **pptr, const u8 *arr, size_t num);
+#define towire_pad_array towire_pad_array_orig
+#include "../towire.c"
+#undef towire_pad_array
+
+#include "../fromwire.c"
+#include <ccan/structeq/structeq.h>
+#include <assert.h>
+#include <stdio.h>
+
+secp256k1_context *secp256k1_ctx;
+
+/* We allow non-zero padding for testing. */
+void towire_pad_array(u8 **pptr, const u8 *arr, size_t num)
+{
+	towire_u8_array(pptr, arr, num);
+}
+
+/* memsetting pubkeys doesn't work */
+static void set_pubkey(struct pubkey *key)
+{
+	u8 der[PUBKEY_DER_LEN];
+	memset(der, 2, sizeof(der));
+	assert(pubkey_from_der(secp256k1_ctx, der, sizeof(der), key));
+}
+
+/* Size up to field. */
+#define upto_field(p, field)				\
+	((char *)&(p)->field - (char *)(p))
+
+/* Size including field. */
+#define with_field(p, field)				\
+	(upto_field((p), field) + sizeof((p)->field))
+
+/* Equal upto this field */
+#define eq_upto(p1, p2, field)			\
+	(memcmp((p1), (p2), upto_field(p1, field)) == 0)
+
+/* Equal upto and including this field */
+#define eq_with(p1, p2, field)			\
+	(memcmp((p1), (p2), with_field(p1, field)) == 0)
+
+/* Equal from fields first to last inclusive. */
+#define eq_between(p1, p2, first, last)					\
+	(memcmp((char *)(p1) + upto_field((p1), first),			\
+		(char *)(p2) + upto_field((p1), first),			\
+		with_field(p1, last) - upto_field(p1, first)) == 0)
+
+/* Equal in one field. */
+#define eq_field(p1, p2, field)						\
+	(memcmp((char *)(p1) + upto_field((p1), field),			\
+		(char *)(p2) + upto_field((p1), field),			\
+		sizeof((p1)->field)) == 0)
+
+#define eq_var(p1, p2, lenfield, field)			\
+	(memcmp((p1)->field, (p2)->field, (p1)->lenfield * sizeof(*(p1)->field)) == 0)
+
+static inline bool eq_skip_(const void *p1, const void *p2,
+			    size_t off, size_t skip, size_t total)
+{
+	if (memcmp(p1, p2, off) != 0)
+		return false;
+	p1 = (char *)p1 + off + skip;
+	p2 = (char *)p2 + off + skip;
+	return memcmp(p1, p2, total - (off + skip)) == 0;
+}
+
+static bool channel_announcement_eq(const struct msg_channel_announcement *a,
+				    const struct msg_channel_announcement *b)
+{
+	return structeq(a, b);
+}
+
+static bool funding_locked_eq(const struct msg_funding_locked *a,
+			      const struct msg_funding_locked *b)
+{
+	return structeq(a, b);
+}
+
+static bool update_fail_htlc_eq(const struct msg_update_fail_htlc *a,
+				const struct msg_update_fail_htlc *b)
+{
+	return eq_with(a, b, reason);
+}
+
+static bool commit_sig_eq(const struct msg_commit_sig *a,
+			  const struct msg_commit_sig *b)
+{
+	return eq_with(a, b, num_htlcs)
+		&& eq_var(a, b, num_htlcs, htlc_signature);
+}
+
+static bool funding_signed_eq(const struct msg_funding_signed *a,
+			      const struct msg_funding_signed *b)
+{
+	return structeq(a, b);
+}
+
+static bool closing_signed_eq(const struct msg_closing_signed *a,
+			      const struct msg_closing_signed *b)
+{
+	return structeq(a, b);
+}
+
+static bool update_fulfill_htlc_eq(const struct msg_update_fulfill_htlc *a,
+				   const struct msg_update_fulfill_htlc *b)
+{
+	return structeq(a, b);
+}
+
+static bool error_eq(const struct msg_error *a,
+		     const struct msg_error *b)
+{
+	return eq_with(a, b, len)
+		&& eq_var(a, b, len, data);
+}
+
+static bool init_eq(const struct msg_init *a,
+		    const struct msg_init *b)
+{
+	return eq_field(a, b, gflen)
+		&& eq_var(a, b, gflen, globalfeatures)
+		&& eq_field(a, b, lflen)
+		&& eq_var(a, b, lflen, localfeatures);
+}
+
+static bool update_fee_eq(const struct msg_update_fee *a,
+			  const struct msg_update_fee *b)
+{
+	return structeq(a, b);
+}
+
+static bool shutdown_eq(const struct msg_shutdown *a,
+			const struct msg_shutdown *b)
+{
+	return eq_with(a, b, len)
+		&& eq_var(a, b, len, scriptpubkey);
+}
+
+static bool funding_created_eq(const struct msg_funding_created *a,
+			       const struct msg_funding_created *b)
+{
+	return eq_with(a, b, output_index)
+		&& eq_field(a, b, signature);
+}
+
+static bool revoke_and_ack_eq(const struct msg_revoke_and_ack *a,
+			      const struct msg_revoke_and_ack *b)
+{
+	return eq_with(a, b, padding)
+		&& eq_field(a, b, num_htlc_timeouts)
+		&& eq_var(a, b, num_htlc_timeouts, htlc_timeout_signature);
+}
+
+static bool open_channel_eq(const struct msg_open_channel *a,
+			    const struct msg_open_channel *b)
+{
+	return eq_with(a, b, max_accepted_htlcs)
+		&& eq_between(a, b, funding_pubkey, first_per_commitment_point);
+}
+
+static bool channel_update_eq(const struct msg_channel_update *a,
+			      const struct msg_channel_update *b)
+{
+	return structeq(a, b);
+}
+
+static bool accept_channel_eq(const struct msg_accept_channel *a,
+			      const struct msg_accept_channel *b)
+{
+	return eq_with(a, b, max_accepted_htlcs)
+		&& eq_between(a, b, funding_pubkey, first_per_commitment_point);
+}
+
+static bool update_add_htlc_eq(const struct msg_update_add_htlc *a,
+			       const struct msg_update_add_htlc *b)
+{
+	return eq_with(a, b, onion_routing_packet);
+}
+
+static bool node_announcement_eq(const struct msg_node_announcement *a,
+				 const struct msg_node_announcement *b)
+{
+	return eq_with(a, b, port)
+		&& eq_between(a, b, node_id, pad)
+		&& eq_field(a, b, alias);
+}
+
+/* Try flipping each bit, try running short. */
+#define test_corruption(a, b, type)				\
+	for (i = 0; i < tal_count(msg) * 8; i++) {		\
+		len = tal_count(msg);				\
+		msg[i / 8] ^= (1 << (i%8));			\
+		b = fromwire_##type(ctx, msg, &len);		\
+		assert(!b || !type##_eq(a, b));			\
+		msg[i / 8] ^= (1 << (i%8));			\
+	}							\
+	for (i = 0; i < tal_count(msg); i++) {			\
+		len = i;					\
+		b = fromwire_##type(ctx, msg, &len);		\
+		assert(!b);					\
+	}
+
+int main(void)
+{
+	struct msg_channel_announcement ca, *ca2;
+	struct msg_funding_locked fl, *fl2;
+	struct msg_update_fail_htlc ufh, *ufh2;
+	struct msg_commit_sig cs, *cs2;
+	struct msg_funding_signed fs, *fs2;
+	struct msg_closing_signed cls, *cls2;
+	struct msg_update_fulfill_htlc uflh, *uflh2;
+	struct msg_error e, *e2;
+	struct msg_init init, *init2;
+	struct msg_update_fee uf, *uf2;
+	struct msg_shutdown shutdown, *shutdown2;
+	struct msg_funding_created fc, *fc2;
+	struct msg_revoke_and_ack raa, *raa2;
+	struct msg_open_channel oc, *oc2;
+	struct msg_channel_update cu, *cu2;
+	struct msg_accept_channel ac, *ac2;
+	struct msg_update_add_htlc uah, *uah2;
+	struct msg_node_announcement na, *na2;
+	void *ctx = tal(NULL, char);
+	size_t i, len;
+	u8 *msg;
+
+	secp256k1_ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY
+						 | SECP256K1_CONTEXT_SIGN);
+
+	memset(&ca, 2, sizeof(ca));
+	set_pubkey(&ca.node_id_1);
+	set_pubkey(&ca.node_id_2);
+	set_pubkey(&ca.bitcoin_key_1);
+	set_pubkey(&ca.bitcoin_key_2);
+	
+	msg = towire_channel_announcement(ctx, &ca);
+	len = tal_count(msg);
+	ca2 = fromwire_channel_announcement(ctx, msg, &len);
+	assert(len == 0);
+	assert(channel_announcement_eq(&ca, ca2));
+	test_corruption(&ca, ca2, channel_announcement);
+
+	memset(&fl, 2, sizeof(fl));
+	set_pubkey(&fl.next_per_commitment_point);
+	
+	msg = towire_funding_locked(ctx, &fl);
+	len = tal_count(msg);
+	fl2 = fromwire_funding_locked(ctx, msg, &len);
+	assert(len == 0);
+	assert(funding_locked_eq(&fl, fl2));
+	test_corruption(&fl, fl2, funding_locked);
+	
+	memset(&ufh, 2, sizeof(ufh));
+	
+	msg = towire_update_fail_htlc(ctx, &ufh);
+	len = tal_count(msg);
+	ufh2 = fromwire_update_fail_htlc(ctx, msg, &len);
+	assert(len == 0);
+	assert(update_fail_htlc_eq(&ufh, ufh2));
+	test_corruption(&ufh, ufh2, update_fail_htlc);
+
+	memset(&cs, 2, sizeof(cs));
+	cs.num_htlcs = 2;
+	cs.htlc_signature = tal_arr(ctx, struct signature, 2);
+	memset(cs.htlc_signature, 2, sizeof(struct signature)*2);
+	
+	msg = towire_commit_sig(ctx, &cs);
+	len = tal_count(msg);
+	cs2 = fromwire_commit_sig(ctx, msg, &len);
+	assert(len == 0);
+	assert(commit_sig_eq(&cs, cs2));
+	test_corruption(&cs, cs2, commit_sig);
+
+	memset(&fs, 2, sizeof(fs));
+	
+	msg = towire_funding_signed(ctx, &fs);
+	len = tal_count(msg);
+	fs2 = fromwire_funding_signed(ctx, msg, &len);
+	assert(len == 0);
+	assert(funding_signed_eq(&fs, fs2));
+	test_corruption(&fs, fs2, funding_signed);
+
+	memset(&cls, 2, sizeof(cls));
+	
+	msg = towire_closing_signed(ctx, &cls);
+	len = tal_count(msg);
+	cls2 = fromwire_closing_signed(ctx, msg, &len);
+	assert(len == 0);
+	assert(closing_signed_eq(&cls, cls2));
+	test_corruption(&cls, cls2, closing_signed);
+	
+	memset(&uflh, 2, sizeof(uflh));
+	
+	msg = towire_update_fulfill_htlc(ctx, &uflh);
+	len = tal_count(msg);
+	uflh2 = fromwire_update_fulfill_htlc(ctx, msg, &len);
+	assert(len == 0);
+	assert(update_fulfill_htlc_eq(&uflh, uflh2));
+	test_corruption(&uflh, uflh2, update_fulfill_htlc);
+
+	memset(&e, 2, sizeof(e));
+	e.len = 2;
+	e.data = tal_arr(ctx, u8, 2);
+	memset(e.data, 2, 2);
+	
+	msg = towire_error(ctx, &e);
+	len = tal_count(msg);
+	e2 = fromwire_error(ctx, msg, &len);
+	assert(len == 0);
+	assert(error_eq(&e, e2));
+	test_corruption(&e, e2, error);
+
+	memset(&init, 2, sizeof(init));
+	init.gflen = 2;
+	init.globalfeatures = tal_arr(ctx, u8, 2);
+	memset(init.globalfeatures, 2, 2);
+	init.lflen = 2;
+	init.localfeatures = tal_arr(ctx, u8, 2);
+	memset(init.localfeatures, 2, 2);
+
+	msg = towire_init(ctx, &init);
+	len = tal_count(msg);
+	init2 = fromwire_init(ctx, msg, &len);
+	assert(len == 0);
+	assert(init_eq(&init, init2));
+	test_corruption(&init, init2, init);
+
+	memset(&uf, 2, sizeof(uf));
+	
+	msg = towire_update_fee(ctx, &uf);
+	len = tal_count(msg);
+	uf2 = fromwire_update_fee(ctx, msg, &len);
+	assert(len == 0);
+	assert(update_fee_eq(&uf, uf2));
+	test_corruption(&uf, uf2, update_fee);
+
+	memset(&shutdown, 2, sizeof(shutdown));
+	shutdown.len = 2;
+	shutdown.scriptpubkey = tal_arr(ctx, u8, 2);
+	memset(shutdown.scriptpubkey, 2, 2);
+	
+	msg = towire_shutdown(ctx, &shutdown);
+	len = tal_count(msg);
+	shutdown2 = fromwire_shutdown(ctx, msg, &len);
+	assert(len == 0);
+	assert(shutdown_eq(&shutdown, shutdown2));
+	test_corruption(&shutdown, shutdown2, shutdown);
+	
+	memset(&fc, 2, sizeof(fc));
+	
+	msg = towire_funding_created(ctx, &fc);
+	len = tal_count(msg);
+	fc2 = fromwire_funding_created(ctx, msg, &len);
+	assert(len == 0);
+	assert(funding_created_eq(&fc, fc2));
+	test_corruption(&fc, fc2, funding_created);
+
+	memset(&raa, 2, sizeof(raa));
+	set_pubkey(&raa.next_per_commitment_point);
+	raa.num_htlc_timeouts = 2;
+	raa.htlc_timeout_signature = tal_arr(ctx, struct signature, 2);
+	memset(raa.htlc_timeout_signature, 2, sizeof(struct signature) * 2);
+	
+	msg = towire_revoke_and_ack(ctx, &raa);
+	len = tal_count(msg);
+	raa2 = fromwire_revoke_and_ack(ctx, msg, &len);
+	assert(len == 0);
+	assert(revoke_and_ack_eq(&raa, raa2));
+	test_corruption(&raa, raa2, revoke_and_ack);
+
+	memset(&oc, 2, sizeof(oc));
+	set_pubkey(&oc.funding_pubkey);
+	set_pubkey(&oc.revocation_basepoint);
+	set_pubkey(&oc.payment_basepoint);
+	set_pubkey(&oc.delayed_payment_basepoint);
+	set_pubkey(&oc.first_per_commitment_point);
+	
+	msg = towire_open_channel(ctx, &oc);
+	len = tal_count(msg);
+	oc2 = fromwire_open_channel(ctx, msg, &len);
+	assert(len == 0);
+	assert(open_channel_eq(&oc, oc2));
+	test_corruption(&oc, oc2, open_channel);
+
+	memset(&cu, 2, sizeof(cu));
+	
+	msg = towire_channel_update(ctx, &cu);
+	len = tal_count(msg);
+	cu2 = fromwire_channel_update(ctx, msg, &len);
+	assert(len == 0);
+	assert(channel_update_eq(&cu, cu2));
+	test_corruption(&cu, cu2, channel_update);
+
+	memset(&ac, 2, sizeof(ac));
+	set_pubkey(&ac.funding_pubkey);
+	set_pubkey(&ac.revocation_basepoint);
+	set_pubkey(&ac.payment_basepoint);
+	set_pubkey(&ac.delayed_payment_basepoint);
+	set_pubkey(&ac.first_per_commitment_point);
+	
+	msg = towire_accept_channel(ctx, &ac);
+	len = tal_count(msg);
+	ac2 = fromwire_accept_channel(ctx, msg, &len);
+	assert(len == 0);
+	assert(accept_channel_eq(&ac, ac2));
+	test_corruption(&ac, ac2, accept_channel);
+
+	memset(&uah, 2, sizeof(uah));
+	
+	msg = towire_update_add_htlc(ctx, &uah);
+	len = tal_count(msg);
+	uah2 = fromwire_update_add_htlc(ctx, msg, &len);
+	assert(len == 0);
+	assert(update_add_htlc_eq(&uah, uah2));
+	test_corruption(&uah, uah2, update_add_htlc);
+
+	memset(&na, 2, sizeof(na));
+	set_pubkey(&na.node_id);
+
+	msg = towire_node_announcement(ctx, &na);
+	len = tal_count(msg);
+	na2 = fromwire_node_announcement(ctx, msg, &len);
+	assert(len == 0);
+	assert(node_announcement_eq(&na, na2));
+	test_corruption(&na, na2, node_announcement);
+
+	/* No memory leaks please */
+	secp256k1_context_destroy(secp256k1_ctx);
+	tal_free(ctx);
+	return 0;
+}
--- a/wire/tools/generate-wire.py
+++ b/wire/tools/generate-wire.py
@ -0,0 +1,265 @@
+#! /usr/bin/python3
+# Read from stdin, spit out C header or body.
+
+from optparse import OptionParser
+from collections import namedtuple
+import fileinput
+import re
+
+Enumtype = namedtuple('Enumtype', ['name', 'value'])
+
+class Field(object):
+    def __init__(self,message,name,size):
+        self.message = message
+        self.name = name.replace('-', '_')
+        (self.typename, self.basesize) = Field._guess_type(message,self.name,size)
+
+        try:
+            if int(size) % self.basesize != 0:
+                raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(size,self.message,self.name,self.basesize))
+            self.num_elems = int(int(size) / self.basesize)
+        except ValueError:
+            self.num_elems = 0
+            # If it's a multiplicitive expression, must end in basesize.
+            if '*' in size:
+                tail='*' + str(self.basesize)
+                if not size.endswith(tail):
+                    raise ValueError('Invalid size {} for {}.{} not a multiple of {}'.format(size,self.message,self.name,self.basesize))
+                size = size[:-len(tail)]
+            else:
+                if self.basesize != 1:
+                    raise ValueError('Invalid size {} for {}.{} not expressed as a multiple of {}'.format(size,self.message,self.name,self.basesize))
+
+            self.lenvar = size.replace('-','_')
+
+    def is_padding(self):
+        return self.name.startswith('pad')
+
+    # Padding is always treated as an array.
+    def is_array(self):
+        return self.num_elems > 1 or self.is_padding()
+
+    def is_variable_size(self):
+        return self.num_elems == 0
+
+    def is_assignable(self):
+        if self.is_array() or self.is_variable_size():
+            return False
+        return self.typename == 'u8' or self.typename == 'u16' or self.typename == 'u32' or self.typename == 'u64'
+
+    # Returns typename and base size
+    @staticmethod
+    def _guess_type(message, fieldname, sizestr):
+        if fieldname.startswith('pad'):
+            return ('pad',1)
+
+        if fieldname.endswith('channel_id'):
+            return ('struct channel_id',8)
+
+        if message == 'node_announcement' and fieldname == 'ipv6':
+            return ('struct ipv6',16)
+
+        if fieldname.endswith('features'):
+            return ('u8',1)
+    
+        # We translate signatures and pubkeys.
+        if 'signature' in fieldname:
+            return ('struct signature',64)
+
+        # The remainder should be fixed sizes.
+        if sizestr == '33':
+            return ('struct pubkey',33)
+        if sizestr == '32':
+            return ('struct sha256',32)
+        if sizestr == '8':
+            return ('u64',8)
+        if sizestr == '4':
+            return ('u32',4)
+        if sizestr == '2':
+            return ('u16',2)
+        if sizestr == '1':
+            return ('u8',1)
+
+        # We whitelist specific things here, otherwise we'd treat everything
+        # as a u8 array.
+        if message == 'update_fail_htlc' and fieldname == 'reason':
+            return ('u8', 1)
+        if message == 'update_add_htlc' and fieldname == 'onion_routing_packet':
+            return ('u8', 1)
+        if message == 'node_announcement' and fieldname == 'alias':
+            return ('u8',1)
+        if message == 'error' and fieldname == 'data':
+            return ('u8',1)
+        if message == 'shutdown' and fieldname == 'scriptpubkey':
+            return ('u8',1)
+        if message == 'node_announcement' and fieldname == 'rgb_color':
+            return ('u8',1)
+    
+        raise ValueError('Unknown size {} for {}'.format(sizestr,fieldname))
+
+class Message(object):
+    def __init__(self,name,enum):
+        self.name = name
+        self.enum = enum
+        self.fields = []
+
+    def checkLenField(self,field):
+        for f in self.fields:
+            if f.name == field.lenvar:
+                if f.typename != 'u16':
+                    raise ValueError('Field {} has non-u16 length variable {}'
+                                     .format(field.name, field.lenvar))
+
+                if f.is_array() or f.is_variable_size():
+                    raise ValueError('Field {} has non-simple length variable {}'
+                                     .format(field.name, field.lenvar))
+                return
+        raise ValueError('Field {} unknown length variable {}'
+                         .format(field.name, field.lenvar))
+
+    def addField(self,field):
+        # We assume field lengths are 16 bit, to avoid overflow issues and
+        # massive allocations.
+        if field.is_variable_size():
+            self.checkLenField(field)
+        self.fields.append(field)
+
+    def print_structure(self):
+        print('struct msg_{} {{'.format(self.name));
+
+        for f in self.fields:
+            # If size isn't known, it's a pointer.
+            if f.is_array():
+                print('\t{} {}[{}];'.format(f.typename, f.name, f.num_elems))
+            elif f.is_variable_size():
+                print('\t{} *{};'.format(f.typename, f.name))
+            else:
+                print('\t{} {};'.format(f.typename, f.name))
+
+        print('};')
+
+    def print_fromwire(self,is_header):
+        print('struct msg_{} *fromwire_{}(const tal_t *ctx, const void *p, size_t *len)'.format(self.name,self.name), end='')
+
+        if is_header:
+            print(';')
+            return
+
+        print('\n'
+              '{{\n'
+              '\tconst u8 *cursor = p;\n'
+              '\tstruct msg_{} *in = tal(ctx, struct msg_{});\n'
+              ''.format(self.name, self.name));
+
+        for f in self.fields:
+            basetype=f.typename
+            if f.typename.startswith('struct '):
+                basetype=f.typename[7:]
+
+            if f.is_array():
+                print('\tfromwire_{}_array(&cursor, len, in->{}, {});'
+                      .format(basetype, f.name, f.num_elems))
+            elif f.is_variable_size():
+                print('\tin->{} = tal_arr(in, {}, in->{});'
+                      .format(f.name, f.typename, f.lenvar))
+                print('\tfromwire_{}_array(&cursor, len, in->{}, in->{});'
+                      .format(basetype, f.name, f.lenvar))
+            elif f.is_assignable():
+                print('\tin->{} = fromwire_{}(&cursor, len);'
+                      .format(f.name, basetype))
+            else:
+                print('\tfromwire_{}(&cursor, len, &in->{});'
+                      .format(basetype, f.name))
+
+        print('\n'
+              '\tif (!cursor)\n'
+              '\t\treturn tal_free(in);\n'
+              '\treturn in;\n'
+              '}\n')
+   
+    def print_towire(self,is_header):
+        print('u8 *towire_{}(const tal_t *ctx, const struct msg_{} *out)'.format(self.name,self.name), end='')
+
+        if is_header:
+            print(';')
+            return
+    
+        print('\n'
+              '{\n'
+              '\tu8 *p = tal_arr(ctx, u8, 0);\n'
+              '')
+
+        for f in self.fields:
+            basetype=f.typename
+            if f.typename.startswith('struct '):
+                basetype=f.typename[7:]
+
+            if f.is_array():
+                print('\ttowire_{}_array(&p, out->{}, {});'
+                      .format(basetype, f.name, f.num_elems))
+            elif f.is_variable_size():
+                print('\ttowire_{}_array(&p, out->{}, out->{});'
+                      .format(basetype, f.name, f.lenvar))
+            elif f.is_assignable():
+                print('\ttowire_{}(&p, out->{});'
+                      .format(basetype, f.name))
+            else:
+                print('\ttowire_{}(&p, &out->{});'
+                      .format(basetype, f.name))
+
+        print('\n'
+              '\treturn p;\n'
+              '}\n')
+   
+parser = OptionParser()
+parser.add_option("--header",
+                  action="store_true", dest="output_header", default=False,
+                  help="Create gen_wire.h")
+
+(options, args) = parser.parse_args()
+
+if options.output_header:
+    print('#ifndef LIGHTNING_WIRE_GEN_WIRE_H\n'
+          '#define LIGHTNING_WIRE_GEN_WIRE_H\n'
+          '#include <ccan/tal/tal.h>\n'
+          '#include <wire/wire.h>\n'
+          '\n'
+          'typedef u8 pad;\n'
+          '')
+else:
+    print('#include "gen_wire.h"\n'
+          '')
+
+# Maps message names to messages
+messages = { }
+
+# Read csv lines.  Single comma is the message values, more is offset/len.
+for line in fileinput.input(args):
+    parts = line.rstrip().split(',')
+
+    if len(parts) == 2:
+        # eg commit_sig,132
+        messages[parts[0]] = Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), int(parts[1])))
+    else:
+        # eg commit_sig,0,channel-id,8
+        messages[parts[0]].addField(Field(parts[0], parts[2], parts[3]))
+
+if options.output_header:
+    # Dump out enum, sorted by value order.
+    print('enum wire_type {')
+    for m in sorted(messages.values(),key=lambda x:x.enum.value):
+        print('\t{} = {},'.format(m.enum.name, m.enum.value))
+    print('};')
+
+    # Dump out structure definitions.
+    for m in messages.values():
+        m.print_structure()
+
+for m in messages.values():
+    m.print_fromwire(options.output_header)
+
+for m in messages.values():
+    m.print_towire(options.output_header)
+    
+if options.output_header:
+    print('#endif /* LIGHTNING_WIRE_GEN_WIRE_H */\n')
--- a/wire/towire.c
+++ b/wire/towire.c
@ -0,0 +1,97 @@
+#include "wire.h"
+#include <ccan/endian/endian.h>
+#include <ccan/mem/mem.h>
+#include <ccan/tal/tal.h>
+
+void towire(u8 **pptr, const void *data, size_t len)
+{
+	size_t oldsize = tal_count(*pptr);
+
+	tal_resize(pptr, oldsize + len);
+	memcpy(*pptr + oldsize, memcheck(data, len), len);
+}
+
+void towire_u8(u8 **pptr, u8 v)
+{
+	towire(pptr, &v, sizeof(v));
+}
+
+void towire_u16(u8 **pptr, u16 v)
+{
+	be16 l = cpu_to_be16(v);
+	towire(pptr, &l, sizeof(l));
+}
+
+void towire_u32(u8 **pptr, u32 v)
+{
+	be32 l = cpu_to_be32(v);
+	towire(pptr, &l, sizeof(l));
+}
+
+void towire_u64(u8 **pptr, u64 v)
+{
+	be64 l = cpu_to_be64(v);
+	towire(pptr, &l, sizeof(l));
+}
+
+void towire_pubkey(u8 **pptr, const struct pubkey *pubkey)
+{
+	u8 output[PUBKEY_DER_LEN];
+	size_t outputlen = sizeof(output);
+
+	secp256k1_ec_pubkey_serialize(secp256k1_ctx, output, &outputlen,
+				      &pubkey->pubkey,
+				      SECP256K1_EC_COMPRESSED);
+	towire(pptr, output, outputlen);
+}
+
+void towire_signature(u8 **pptr, const struct signature *sig)
+{
+	u8 compact[64];
+
+	secp256k1_ecdsa_signature_serialize_compact(secp256k1_ctx,
+						    compact, &sig->sig);
+	towire(pptr, compact, sizeof(compact));
+}
+
+void towire_channel_id(u8 **pptr, const struct channel_id *channel_id)
+{
+	be32 txnum = cpu_to_be32(channel_id->txnum);
+	u8 outnum = channel_id->outnum;
+	
+	towire_u32(pptr, channel_id->blocknum);
+	towire(pptr, (char *)&txnum + 1, 3);
+	towire(pptr, &outnum, 1);
+}
+
+void towire_sha256(u8 **pptr, const struct sha256 *sha256)
+{
+	towire(pptr, sha256, sizeof(*sha256));
+}
+
+void towire_ipv6(u8 **pptr, const struct ipv6 *ipv6)
+{
+	towire(pptr, ipv6, sizeof(*ipv6));
+}
+
+void towire_u8_array(u8 **pptr, const u8 *arr, size_t num)
+{
+	towire(pptr, arr, num);
+}
+
+void towire_pad_array(u8 **pptr, const u8 *arr, size_t num)
+{
+	/* Simply insert zeros. */
+	size_t oldsize = tal_count(*pptr);
+
+	tal_resize(pptr, oldsize + num);
+	memset(*pptr + oldsize, 0, num);
+}
+	
+void towire_signature_array(u8 **pptr, const struct signature *arr, size_t num)
+{
+	size_t i;
+
+	for (i = 0; i < num; i++)
+		towire_signature(pptr, arr+i);
+}
--- a/wire/wire.h
+++ b/wire/wire.h
@ -0,0 +1,60 @@
+#ifndef LIGHTNING_WIRE_WIRE_H
+#define LIGHTNING_WIRE_WIRE_H
+#include "config.h"
+#include <bitcoin/pubkey.h>
+#include <bitcoin/signature.h>
+#include <ccan/crypto/sha256/sha256.h>
+#include <ccan/short_types/short_types.h>
+#include <stdlib.h>
+
+/* FIXME: Move this declaration! */
+extern secp256k1_context *secp256k1_ctx;
+
+struct pubkey;
+struct sha256;
+struct channel_id {
+	u32 blocknum;
+	u32 txnum : 24;
+	u8 outnum : 8;
+};
+struct ipv6 {
+	u8 addr[16];
+};
+
+void towire(u8 **pptr, const void *data, size_t len);
+void towire_pubkey(u8 **pptr, const struct pubkey *pubkey);
+void towire_signature(u8 **pptr, const struct signature *signature);
+void towire_channel_id(u8 **pptr, const struct channel_id *channel_id);
+void towire_sha256(u8 **pptr, const struct sha256 *sha256);
+void towire_ipv6(u8 **pptr, const struct ipv6 *ipv6);
+void towire_u8(u8 **pptr, u8 v);
+void towire_u16(u8 **pptr, u16 v);
+void towire_u32(u8 **pptr, u32 v);
+void towire_u64(u8 **pptr, u64 v);
+
+void towire_u8_array(u8 **pptr, const u8 *arr, size_t num);
+void towire_pad_array(u8 **pptr, const u8 *arr, size_t num);
+void towire_signature_array(u8 **pptr, const struct signature *arr, size_t num);
+
+
+const u8 *fromwire(const u8 **cursor, size_t *max, void *copy, size_t n);
+u8 fromwire_u8(const u8 **cursor, size_t *max);
+u16 fromwire_u16(const u8 **cursor, size_t *max);
+u32 fromwire_u32(const u8 **cursor, size_t *max);
+u64 fromwire_u64(const u8 **cursor, size_t *max);
+void fromwire_pubkey(const u8 **cursor, size_t *max, struct pubkey *pubkey);
+void fromwire_signature(const u8 **cursor, size_t *max,
+			struct signature *signature);
+void fromwire_channel_id(const u8 **cursor, size_t *max,
+			 struct channel_id *channel_id);
+void fromwire_sha256(const u8 **cursor, size_t *max, struct sha256 *sha256);
+void fromwire_ipv6(const u8 **cursor, size_t *max, struct ipv6 *ipv6);
+
+void fromwire_u8_array(const u8 **cursor, size_t *max,
+		       u8 *arr, size_t num);
+void fromwire_pad_array(const u8 **cursor, size_t *max,
+			u8 *arr, size_t num);
+void fromwire_signature_array(const u8 **cursor, size_t *max,
+			      struct signature *arr, size_t num);
+
+#endif /* LIGHTNING_WIRE_WIRE_H */