bolt11: handle `9` fields for new features.

This implements https://github.com/lightningnetwork/lightning-rfc/pull/656

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - JSON API: `listfunds` now lists a blockheight for confirmed transactions
 
+- bolt11: support for parsing feature bits (field `9`).
+
 ### Changed
 
 - JSON API: `txprepare` now uses `outputs` as parameter other than `destination` and `satoshi`

common/bolt11.c

@@ -9,6 +9,7 @@
 #include <common/bech32.h>
 #include <common/bech32_util.h>
 #include <common/bolt11.h>
+#include <common/features.h>
 #include <common/utils.h>
 #include <errno.h>
 #include <hsmd/gen_hsm_wire.h>
@@ -431,6 +432,59 @@ static char *decode_r(struct bolt11 *b11,
         return NULL;
 }
 
+static void shift_bitmap_down(u8 *bitmap, size_t bits)
+{
+	u8 prev = 0;
+	assert(bits < CHAR_BIT);
+
+	for (size_t i = 0; i < tal_bytelen(bitmap); i++) {
+		/* Save top bits for next one */
+		u8 v = bitmap[i];
+		bitmap[i] = (prev | (v >> bits));
+		prev = (v << (8 - bits));
+	}
+	assert(prev == 0);
+}
+
+/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+ *
+ * `9` (5): `data_length` variable. One or more bytes containing features
+ *  supported or required for receiving this payment.
+ *  See [Feature Bits](#feature-bits).
+ */
+static char *decode_9(struct bolt11 *b11,
+                      struct hash_u5 *hu5,
+                      u5 **data, size_t *data_len,
+                      size_t data_length)
+{
+        size_t flen = (data_length * 5 + 7) / 8;
+
+        b11->features = tal_arr(b11, u8, flen);
+        pull_bits_certain(hu5, data, data_len, b11->features,
+			  data_length * 5, true);
+
+	/* pull_bits pads with zero bits: we need to remove them. */
+	shift_bitmap_down(b11->features,
+			  flen * 8 - data_length * 5);
+
+	/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+	 *
+	 * - if the `9` field contains unknown _odd_ bits that are non-zero:
+	 *   - MUST ignore the bit.
+	 * - if the `9` field contains unknown _even_ bits that are non-zero:
+	 *   - MUST fail.
+	 */
+	/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+	 * The field is big-endian.  The least-significant bit is numbered 0,
+	 * which is _even_, and the next most significant bit is numbered 1,
+	 * which is _odd_. */
+	for (size_t i = 0; i < data_length * 5; i += 2)
+		if (feature_is_set(b11->features, i))
+			return tal_fmt(b11, "9: unknown feature bit %zu", i);
+
+	return NULL;
+}
+
 struct bolt11 *new_bolt11(const tal_t *ctx,
 			  const struct amount_msat *msat TAKES)
 {
@@ -443,6 +497,7 @@ struct bolt11 *new_bolt11(const tal_t *ctx,
         b11->routes = NULL;
         b11->msat = NULL;
         b11->expiry = DEFAULT_X;
+	b11->features = tal_arr(b11, u8, 0);
         b11->min_final_cltv_expiry = DEFAULT_C;
 
         if (msat)
@@ -634,6 +689,10 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str,
                         problem = decode_r(b11, &hu5, &data, &data_len,
                                            data_length);
                         break;
+		case '9':
+			problem = decode_9(b11, &hu5, &data, &data_len,
+					   data_length);
+			break;
                 default:
                         unknown_field(b11, &hu5, &data, &data_len,
                                       bech32_charset[type], data_length);
@@ -732,11 +791,16 @@ static void push_varlen_uint(u5 **data, u64 val, size_t nbits)
  * 1. `data_length` (10 bits, big-endian)
  * 1. `data` (`data_length` x 5 bits)
  */
-static void push_field(u5 **data, char type, const void *src, size_t nbits)
+static void push_field_type_and_len(u5 **data, char type, size_t nbits)
 {
         assert(bech32_charset_rev[(unsigned char)type] >= 0);
         push_varlen_uint(data, bech32_charset_rev[(unsigned char)type], 5);
         push_varlen_uint(data, (nbits + 4) / 5, 10);
+}
+
+static void push_field(u5 **data, char type, const void *src, size_t nbits)
+{
+	push_field_type_and_len(data, type, nbits);
         bech32_push_bits(data, src, nbits);
 }
 
@@ -849,6 +913,31 @@ static void encode_r(u5 **data, const struct route_info *r)
         tal_free(rinfo);
 }
 
+static void maybe_encode_9(u5 **data, const u8 *features)
+{
+	u5 *f5 = tal_arr(NULL, u5, 0);
+
+	for (size_t i = 0; i < tal_count(features) * CHAR_BIT; i++) {
+		if (!feature_is_set(features, i))
+			continue;
+		/* We expand it out so it makes a BE 5-bit/btye bitfield */
+		set_feature_bit(&f5, (i / 5) * 8 + (i % 5));
+	}
+
+	/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+	 *
+	 * - if `9` contains non-zero bits:
+	 *   - SHOULD use the minimum `data_length` possible.
+	 * - otherwise:
+	 *   - MUST omit the `9` field altogether.
+	 */
+	if (tal_count(f5) != 0) {
+		push_field_type_and_len(data, '9', tal_count(f5) * 5);
+                tal_expand(data, f5, tal_count(f5));
+	}
+	tal_free(f5);
+}
+
 static bool encode_extra(u5 **data, const struct bolt11_field *extra)
 {
         size_t len;
@@ -952,6 +1041,8 @@ char *bolt11_encode_(const tal_t *ctx,
         for (size_t i = 0; i < tal_count(b11->routes); i++)
                 encode_r(&data, b11->routes[i]);
 
+	maybe_encode_9(&data, b11->features);
+
         list_for_each(&b11->extra_fields, extra, list)
                 if (!encode_extra(&data, extra))
 			return NULL;

common/bolt11.h

@@ -64,6 +64,9 @@ struct bolt11 {
 	/* signature of sha256 of entire thing. */
 	secp256k1_ecdsa_signature sig;
 
+	/* Features bitmap, if any. */
+	u8 *features;
+
 	struct list_head extra_fields;
 };
 

common/test/run-bolt11.c

@@ -2,6 +2,7 @@
 #include "../bech32.c"
 #include "../bech32_util.c"
 #include "../bolt11.c"
+#include "../features.c"
 #include "../node_id.c"
 #include "../hash_u5.c"
 #include <ccan/err/err.h>
@@ -99,6 +100,8 @@ static void test_b11(const char *b11str,
 	else
 		assert(streq(b11->description, expect_b11->description));
 
+	assert(memeq(b11->features, tal_bytelen(b11->features),
+		     expect_b11->features, tal_bytelen(expect_b11->features)));
 	assert(b11->expiry == expect_b11->expiry);
 	assert(b11->min_final_cltv_expiry == expect_b11->min_final_cltv_expiry);
 
@@ -141,6 +144,7 @@ int main(void)
 	struct amount_msat msatoshi;
 	const char *badstr;
         struct bolt11_field *extra;
+	char *fail;
 
 	wally_init(0);
 	secp256k1_ctx = wally_get_secp_context();
@@ -262,7 +266,6 @@ int main(void)
 	badstr = "lnbc20mpvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7";
 
 	for (size_t i = 0; i <= strlen(badstr); i++) {
-		char *fail;
 		if (bolt11_decode(tmpctx, tal_strndup(tmpctx, badstr, i),
 				  NULL, &fail))
 			abort();
@@ -309,6 +312,54 @@ int main(void)
 
 	test_b11("lntb30m1pw2f2yspp5s59w4a0kjecw3zyexm7zur8l8n4scw674w8sftjhwec33km882gsdpa2pshjmt9de6zqun9w96k2um5ypmkjargypkh2mr5d9cxzun5ypeh2ursdae8gxqruyqvzddp68gup69uhnzwfj9cejuvf3xshrwde68qcrswf0d46kcarfwpshyaplw3skw0tdw4k8g6tsv9e8g4a3hx0v945csrmpm7yxyaamgt2xu7mu4xyt3vp7045n4k4czxf9kj0vw0m8dr5t3pjxuek04rtgyy8uzss5eet5gcyekd6m7u0mzv5sp7mdsag", b11, NULL);
 
+	/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+	 *
+	 * > ### Please send $30 for coffee beans to the same peer, which supports features 1 and 9
+	 * > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl
+	 *
+	 * Breakdown:
+	 *
+	 * * `lnbc`: prefix, Lightning on Bitcoin mainnet
+	 * * `25m`: amount (25 milli-bitcoin)
+	 * * `1`: Bech32 separator
+	 * * `pvjluez`: timestamp (1496314658)
+	 * * `p`: payment hash...
+	 * * `d`: short description
+	 *   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
+	 *   * `vdhkven9v5sxyetpdees`: 'coffee beans'
+	 * * `9`: features
+	 *   * `qz`: `data_length` (`q` = 0, `z` = 2; 0 * 32 + 2 == 2)
+	 *   * `sz`: b1000000010
+	 * * `e992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq`: signature
+	 * * `73t7cl`: Bech32 checksum
+	 */
+	msatoshi = AMOUNT_MSAT(25 * (1000ULL * 100000000) / 1000);
+	b11 = new_bolt11(tmpctx, &msatoshi);
+	b11->chain = chainparams_for_network("bitcoin");
+	b11->timestamp = 1496314658;
+	if (!hex_decode("0001020304050607080900010203040506070809000102030405060708090102",
+			strlen("0001020304050607080900010203040506070809000102030405060708090102"),
+			&b11->payment_hash, sizeof(b11->payment_hash)))
+		abort();
+	b11->receiver_id = node;
+	b11->description = "coffee beans";
+	set_feature_bit(&b11->features, 1);
+	set_feature_bit(&b11->features, 9);
+
+	test_b11("lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl", b11, NULL);
+
+	/* BOLT-a76d61dc9893eec75b2e9c4a361354c356c46894 #11:
+	 *
+	 * > # Same, but using invalid unknown feature 100
+	 * > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7
+	 */
+	/* This one can be encoded, but not decoded */
+	set_feature_bit(&b11->features, 100);
+	badstr = bolt11_encode(tmpctx, b11, false, test_sign, NULL);
+	assert(streq(badstr, "lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7"));
+	assert(!bolt11_decode(tmpctx, badstr, NULL, &fail));
+	assert(streq(fail, "9: unknown feature bit 100"));
+
 	/* FIXME: Test the others! */
 	wally_cleanup(0);
 	tal_free(tmpctx);

devtools/Makefile

@@ -15,6 +15,7 @@ DEVTOOLS_COMMON_OBJS :=				\
 	common/bolt11.o				\
 	common/crypto_state.o			\
 	common/decode_short_channel_ids.o	\
+	common/features.o			\
 	common/hash_u5.o			\
 	common/node_id.o			\
 	common/per_peer_state.o			\

devtools/bolt11-cli.c

@@ -11,6 +11,7 @@
 #include <common/amount.h>
 #include <common/bech32.h>
 #include <common/bolt11.h>
+#include <common/features.h>
 #include <common/type_to_string.h>
 #include <common/version.h>
 #include <inttypes.h>
@@ -116,7 +117,14 @@ int main(int argc, char *argv[])
 		printf("description_hash: %s\n",
 		       tal_hexstr(ctx, b11->description_hash,
 				  sizeof(*b11->description_hash)));
-
+	if (tal_bytelen(b11->features)) {
+		printf("features:");
+		for (size_t i = 0; i < tal_bytelen(b11->features) * CHAR_BIT; i++) {
+			if (feature_is_set(b11->features, i))
+				printf(" %zu", i);
+		}
+		printf("\n");
+	}
 	for (i = 0; i < tal_count(b11->fallbacks); i++) {
                 struct bitcoin_address pkh;
                 struct ripemd160 sh;

lightningd/invoice.c

@@ -1086,6 +1086,8 @@ static struct command_result *json_decodepay(struct command *cmd,
                                 b11->description_hash);
 	json_add_num(response, "min_final_cltv_expiry",
 		     b11->min_final_cltv_expiry);
+	if (b11->features)
+		json_add_hex_talarr(response, "features", b11->features);
         if (tal_count(b11->fallbacks)) {
 		json_array_start(response, "fallbacks");
 		for (size_t i = 0; i < tal_count(b11->fallbacks); i++)

plugins/Makefile

@@ -24,6 +24,7 @@ PLUGIN_COMMON_OBJS :=				\
 	common/bigsize.o			\
 	common/bolt11.o				\
 	common/daemon.o				\
+	common/features.o			\
 	common/hash_u5.o			\
 	common/json.o				\
 	common/json_helpers.o			\

tests/test_pay.py

@@ -792,6 +792,56 @@ def test_decodepay(node_factory):
     assert b11['fallbacks'][0]['type'] == 'P2WSH'
     assert b11['fallbacks'][0]['addr'] == 'bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3'
 
+    # > ### Please send $30 for coffee beans to the same peer, which supports features 1 and 9
+    # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl
+    #
+    # Breakdown:
+    #
+    # * `lnbc`: prefix, Lightning on Bitcoin mainnet
+    # * `25m`: amount (25 milli-bitcoin)
+    # * `1`: Bech32 separator
+    # * `pvjluez`: timestamp (1496314658)
+    # * `p`: payment hash...
+    # * `d`: short description
+    #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
+    #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
+    # * `9`: features
+    #   * `qz`: `data_length` (`q` = 0, `z` = 2; 0 * 32 + 2 == 2)
+    #   * `sz`: b1000000010
+    # * `e992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq`: signature
+    # * `73t7cl`: Bech32 checksum
+    b11 = l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9qzsze992adudgku8p05pstl6zh7av6rx2f297pv89gu5q93a0hf3g7lynl3xq56t23dpvah6u7y9qey9lccrdml3gaqwc6nxsl5ktzm464sq73t7cl')
+    assert b11['currency'] == 'bc'
+    assert b11['msatoshi'] == 25 * 10**11 // 1000
+    assert b11['amount_msat'] == Millisatoshi(25 * 10**11 // 1000)
+    assert b11['created_at'] == 1496314658
+    assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
+    assert b11['description'] == 'coffee beans'
+    assert b11['expiry'] == 3600
+    assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad'
+    assert b11['features'] == '0202'
+
+    # > # Same, but using invalid unknown feature 100
+    # > lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7
+    #
+    # Breakdown:
+    #
+    # * `lnbc`: prefix, Lightning on Bitcoin mainnet
+    # * `25m`: amount (25 milli-bitcoin)
+    # * `1`: Bech32 separator
+    # * `pvjluez`: timestamp (1496314658)
+    # * `p`: payment hash...
+    # * `d`: short description
+    #   * `q5`: `data_length` (`q` = 0, `5` = 20; 0 * 32 + 20 == 20)
+    #   * `vdhkven9v5sxyetpdees`: 'coffee beans'
+    # * `9`: features
+    #   * `q4`: `data_length` (`q` = 0, `4` = 21; 0 * 32 + 21 == 21)
+    #   * `pqqqqqqqqqqqqqqqqqqsz`: b00001...(90 zeroes)...1000000010
+    # * `k3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sp`: signature
+    # * `hzfxz7`: Bech32 checksum
+    with pytest.raises(RpcError, match='unknown feature.*100'):
+        l1.rpc.decodepay('lnbc25m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5vdhkven9v5sxyetpdees9q4pqqqqqqqqqqqqqqqqqqszk3ed62snp73037h4py4gry05eltlp0uezm2w9ajnerhmxzhzhsu40g9mgyx5v3ad4aqwkmvyftzk4k9zenz90mhjcy9hcevc7r3lx2sphzfxz7')
+
     with pytest.raises(RpcError):
         l1.rpc.decodepay('1111111')