From 3882e8bdf74856c26f08b883a13b42df37794973 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 17 Jun 2020 14:25:56 +0930 Subject: [PATCH] pyln.spec.bolt*: make these separate packages, so versions can change indep. They're almost entirely autogenerated, and we use symlinks into the top directory to reduce replication. They can't be under pyln.spec.message, because a package can't also be a namespace. We also add fulltext and desc fields, and exclude our "gen" files from flake8, since the spec quotes contain weird whitespace. Changelog-Added: Python: pyln.spec.bolt{1,2,4,7} packages. Signed-off-by: Rusty Russell --- Makefile | 2 +- .../pyln/proto/message/bolt1/Makefile | 7 - .../pyln/proto/message/bolt1/__init__.py | 16 - .../pyln/proto/message/bolt1/bolt.py | 5 - .../pyln/proto/message/bolt1/csv.py | 35 - .../pyln/proto/message/bolt2/Makefile | 7 - .../pyln/proto/message/bolt2/__init__.py | 16 - .../pyln/proto/message/bolt2/csv.py | 100 -- .../pyln/proto/message/bolt4/Makefile | 7 - .../pyln/proto/message/bolt4/bolt.py | 5 - .../pyln/proto/message/bolt4/csv.py | 55 - .../pyln/proto/message/bolt7/Makefile | 7 - .../pyln/proto/message/bolt7/__init__.py | 16 - .../pyln/proto/message/bolt7/bolt.py | 5 - .../pyln/proto/message/bolt7/csv.py | 83 - contrib/pyln-spec/Makefile | 47 + .../proto/message/bolt2 => pyln-spec}/bolt.py | 2 +- .../bolt1/pyln/spec/bolt1/__init__.py | 1 + .../pyln-spec/bolt1/pyln/spec/bolt1/bolt.py | 1 + .../pyln-spec/bolt1/pyln/spec/bolt1/gen.py | 976 +++++++++++ .../bolt1/pyln/spec/bolt1/gen_version.py | 2 + contrib/pyln-spec/bolt1/requirements.txt | 1 + contrib/pyln-spec/bolt1/setup.py | 23 + .../bolt1}/tests/test_bolt1.py | 2 +- .../bolt2/pyln/spec/bolt2/__init__.py | 1 + .../pyln-spec/bolt2/pyln/spec/bolt2/bolt.py | 1 + .../pyln-spec/bolt2/pyln/spec/bolt2/gen.py | 1471 +++++++++++++++++ .../bolt2/pyln/spec/bolt2/gen_version.py | 2 + contrib/pyln-spec/bolt2/requirements.txt | 1 + contrib/pyln-spec/bolt2/setup.py | 23 + .../bolt2}/tests/test_bolt2.py | 2 +- .../bolt4/pyln/spec/bolt4/__init__.py | 1 + .../pyln-spec/bolt4/pyln/spec/bolt4/bolt.py | 1 + .../pyln-spec/bolt4/pyln/spec/bolt4/gen.py | 1267 ++++++++++++++ .../bolt4/pyln/spec/bolt4/gen_version.py | 2 + contrib/pyln-spec/bolt4/requirements.txt | 1 + contrib/pyln-spec/bolt4/setup.py | 23 + .../bolt4}/tests/test_bolt4.py | 2 +- .../bolt7/pyln/spec/bolt7/__init__.py | 1 + .../pyln-spec/bolt7/pyln/spec/bolt7/bolt.py | 1 + .../pyln-spec/bolt7/pyln/spec/bolt7/gen.py | 1211 ++++++++++++++ .../bolt7/pyln/spec/bolt7/gen_version.py | 2 + contrib/pyln-spec/bolt7/requirements.txt | 1 + contrib/pyln-spec/bolt7/setup.py | 23 + .../bolt7}/tests/test_bolt7.py | 2 +- contrib/pyln-spec/requirements.txt | 1 + .../__init__.py => pyln-spec/subinit.py} | 10 +- 47 files changed, 5098 insertions(+), 373 deletions(-) delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt1/Makefile delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt1/__init__.py delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt1/bolt.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt1/csv.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt2/Makefile delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt2/__init__.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt2/csv.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt4/Makefile delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt4/bolt.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt4/csv.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt7/Makefile delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt7/__init__.py delete mode 100644 contrib/pyln-proto/pyln/proto/message/bolt7/bolt.py delete mode 100755 contrib/pyln-proto/pyln/proto/message/bolt7/csv.py create mode 100755 contrib/pyln-spec/Makefile rename contrib/{pyln-proto/pyln/proto/message/bolt2 => pyln-spec}/bolt.py (81%) create mode 120000 contrib/pyln-spec/bolt1/pyln/spec/bolt1/__init__.py create mode 120000 contrib/pyln-spec/bolt1/pyln/spec/bolt1/bolt.py create mode 100644 contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen.py create mode 100644 contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py create mode 120000 contrib/pyln-spec/bolt1/requirements.txt create mode 100644 contrib/pyln-spec/bolt1/setup.py rename contrib/{pyln-proto => pyln-spec/bolt1}/tests/test_bolt1.py (98%) create mode 120000 contrib/pyln-spec/bolt2/pyln/spec/bolt2/__init__.py create mode 120000 contrib/pyln-spec/bolt2/pyln/spec/bolt2/bolt.py create mode 100644 contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen.py create mode 100644 contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py create mode 120000 contrib/pyln-spec/bolt2/requirements.txt create mode 100644 contrib/pyln-spec/bolt2/setup.py rename contrib/{pyln-proto => pyln-spec/bolt2}/tests/test_bolt2.py (78%) create mode 120000 contrib/pyln-spec/bolt4/pyln/spec/bolt4/__init__.py create mode 120000 contrib/pyln-spec/bolt4/pyln/spec/bolt4/bolt.py create mode 100644 contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen.py create mode 100644 contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py create mode 120000 contrib/pyln-spec/bolt4/requirements.txt create mode 100644 contrib/pyln-spec/bolt4/setup.py rename contrib/{pyln-proto => pyln-spec/bolt4}/tests/test_bolt4.py (78%) create mode 120000 contrib/pyln-spec/bolt7/pyln/spec/bolt7/__init__.py create mode 120000 contrib/pyln-spec/bolt7/pyln/spec/bolt7/bolt.py create mode 100644 contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen.py create mode 100644 contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py create mode 120000 contrib/pyln-spec/bolt7/requirements.txt create mode 100644 contrib/pyln-spec/bolt7/setup.py rename contrib/{pyln-proto => pyln-spec/bolt7}/tests/test_bolt7.py (90%) create mode 100644 contrib/pyln-spec/requirements.txt rename contrib/{pyln-proto/pyln/proto/message/bolt4/__init__.py => pyln-spec/subinit.py} (57%) diff --git a/Makefile b/Makefile index d20c4163f..9161bf0b6 100644 --- a/Makefile +++ b/Makefile @@ -340,7 +340,7 @@ check-markdown: check-spelling: @tools/check-spelling.sh -PYSRC=$(shell git ls-files "*.py") contrib/pylightning/lightning-pay +PYSRC=$(shell git ls-files "*.py" | grep -v /text.py) contrib/pylightning/lightning-pay check-python: @# E501 line too long (N > 79 characters) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt1/Makefile b/contrib/pyln-proto/pyln/proto/message/bolt1/Makefile deleted file mode 100755 index 7992280f2..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt1/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/make - -SPECDIR := ../../../../../../../lightning-rfc - -csv.py: $(SPECDIR)/01-messaging.md Makefile - SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/ "\1",/'; echo ']') > $@ - chmod a+x $@ diff --git a/contrib/pyln-proto/pyln/proto/message/bolt1/__init__.py b/contrib/pyln-proto/pyln/proto/message/bolt1/__init__.py deleted file mode 100644 index 2ba3aceb6..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt1/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .csv import csv -from .bolt import namespace -import sys - -__version__ = '0.0.1' - -__all__ = [ - 'csv', - 'namespace', -] - -mod = sys.modules[__name__] -for d in namespace.subtypes, namespace.tlvtypes, namespace.messagetypes: - for name in d: - setattr(mod, name, d[name]) - __all__.append(name) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt1/bolt.py b/contrib/pyln-proto/pyln/proto/message/bolt1/bolt.py deleted file mode 100644 index 565c41228..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt1/bolt.py +++ /dev/null @@ -1,5 +0,0 @@ -from pyln.proto.message import MessageNamespace -from .csv import csv - - -namespace = MessageNamespace(csv_lines=csv) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt1/csv.py b/contrib/pyln-proto/pyln/proto/message/bolt1/csv.py deleted file mode 100755 index 4c8289992..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt1/csv.py +++ /dev/null @@ -1,35 +0,0 @@ -csv = [ - "msgtype,init,16", - "msgdata,init,gflen,u16,", - "msgdata,init,globalfeatures,byte,gflen", - "msgdata,init,flen,u16,", - "msgdata,init,features,byte,flen", - "msgdata,init,tlvs,init_tlvs,", - "tlvtype,init_tlvs,networks,1", - "tlvdata,init_tlvs,networks,chains,chain_hash,...", - "msgtype,error,17", - "msgdata,error,channel_id,channel_id,", - "msgdata,error,len,u16,", - "msgdata,error,data,byte,len", - "msgtype,ping,18", - "msgdata,ping,num_pong_bytes,u16,", - "msgdata,ping,byteslen,u16,", - "msgdata,ping,ignored,byte,byteslen", - "msgtype,pong,19", - "msgdata,pong,byteslen,u16,", - "msgdata,pong,ignored,byte,byteslen", - "tlvtype,n1,tlv1,1", - "tlvdata,n1,tlv1,amount_msat,tu64,", - "tlvtype,n1,tlv2,2", - "tlvdata,n1,tlv2,scid,short_channel_id,", - "tlvtype,n1,tlv3,3", - "tlvdata,n1,tlv3,node_id,point,", - "tlvdata,n1,tlv3,amount_msat_1,u64,", - "tlvdata,n1,tlv3,amount_msat_2,u64,", - "tlvtype,n1,tlv4,254", - "tlvdata,n1,tlv4,cltv_delta,u16,", - "tlvtype,n2,tlv1,0", - "tlvdata,n2,tlv1,amount_msat,tu64,", - "tlvtype,n2,tlv2,11", - "tlvdata,n2,tlv2,cltv_expiry,tu32,", -] diff --git a/contrib/pyln-proto/pyln/proto/message/bolt2/Makefile b/contrib/pyln-proto/pyln/proto/message/bolt2/Makefile deleted file mode 100755 index 832891543..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt2/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/make - -SPECDIR := ../../../../../../../lightning-rfc - -csv.py: $(SPECDIR)/02-peer-protocol.md Makefile - SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/ "\1",/'; echo ']') > $@ - chmod a+x $@ diff --git a/contrib/pyln-proto/pyln/proto/message/bolt2/__init__.py b/contrib/pyln-proto/pyln/proto/message/bolt2/__init__.py deleted file mode 100644 index 2ba3aceb6..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt2/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .csv import csv -from .bolt import namespace -import sys - -__version__ = '0.0.1' - -__all__ = [ - 'csv', - 'namespace', -] - -mod = sys.modules[__name__] -for d in namespace.subtypes, namespace.tlvtypes, namespace.messagetypes: - for name in d: - setattr(mod, name, d[name]) - __all__.append(name) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt2/csv.py b/contrib/pyln-proto/pyln/proto/message/bolt2/csv.py deleted file mode 100755 index f43d75bbe..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt2/csv.py +++ /dev/null @@ -1,100 +0,0 @@ -csv = [ - "msgtype,open_channel,32", - "msgdata,open_channel,chain_hash,chain_hash,", - "msgdata,open_channel,temporary_channel_id,byte,32", - "msgdata,open_channel,funding_satoshis,u64,", - "msgdata,open_channel,push_msat,u64,", - "msgdata,open_channel,dust_limit_satoshis,u64,", - "msgdata,open_channel,max_htlc_value_in_flight_msat,u64,", - "msgdata,open_channel,channel_reserve_satoshis,u64,", - "msgdata,open_channel,htlc_minimum_msat,u64,", - "msgdata,open_channel,feerate_per_kw,u32,", - "msgdata,open_channel,to_self_delay,u16,", - "msgdata,open_channel,max_accepted_htlcs,u16,", - "msgdata,open_channel,funding_pubkey,point,", - "msgdata,open_channel,revocation_basepoint,point,", - "msgdata,open_channel,payment_basepoint,point,", - "msgdata,open_channel,delayed_payment_basepoint,point,", - "msgdata,open_channel,htlc_basepoint,point,", - "msgdata,open_channel,first_per_commitment_point,point,", - "msgdata,open_channel,channel_flags,byte,", - "msgdata,open_channel,tlvs,open_channel_tlvs,", - "tlvtype,open_channel_tlvs,upfront_shutdown_script,0", - "tlvdata,open_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", - "msgtype,accept_channel,33", - "msgdata,accept_channel,temporary_channel_id,byte,32", - "msgdata,accept_channel,dust_limit_satoshis,u64,", - "msgdata,accept_channel,max_htlc_value_in_flight_msat,u64,", - "msgdata,accept_channel,channel_reserve_satoshis,u64,", - "msgdata,accept_channel,htlc_minimum_msat,u64,", - "msgdata,accept_channel,minimum_depth,u32,", - "msgdata,accept_channel,to_self_delay,u16,", - "msgdata,accept_channel,max_accepted_htlcs,u16,", - "msgdata,accept_channel,funding_pubkey,point,", - "msgdata,accept_channel,revocation_basepoint,point,", - "msgdata,accept_channel,payment_basepoint,point,", - "msgdata,accept_channel,delayed_payment_basepoint,point,", - "msgdata,accept_channel,htlc_basepoint,point,", - "msgdata,accept_channel,first_per_commitment_point,point,", - "msgdata,accept_channel,tlvs,accept_channel_tlvs,", - "tlvtype,accept_channel_tlvs,upfront_shutdown_script,0", - "tlvdata,accept_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", - "msgtype,funding_created,34", - "msgdata,funding_created,temporary_channel_id,byte,32", - "msgdata,funding_created,funding_txid,sha256,", - "msgdata,funding_created,funding_output_index,u16,", - "msgdata,funding_created,signature,signature,", - "msgtype,funding_signed,35", - "msgdata,funding_signed,channel_id,channel_id,", - "msgdata,funding_signed,signature,signature,", - "msgtype,funding_locked,36", - "msgdata,funding_locked,channel_id,channel_id,", - "msgdata,funding_locked,next_per_commitment_point,point,", - "msgtype,shutdown,38", - "msgdata,shutdown,channel_id,channel_id,", - "msgdata,shutdown,len,u16,", - "msgdata,shutdown,scriptpubkey,byte,len", - "msgtype,closing_signed,39", - "msgdata,closing_signed,channel_id,channel_id,", - "msgdata,closing_signed,fee_satoshis,u64,", - "msgdata,closing_signed,signature,signature,", - "msgtype,update_add_htlc,128", - "msgdata,update_add_htlc,channel_id,channel_id,", - "msgdata,update_add_htlc,id,u64,", - "msgdata,update_add_htlc,amount_msat,u64,", - "msgdata,update_add_htlc,payment_hash,sha256,", - "msgdata,update_add_htlc,cltv_expiry,u32,", - "msgdata,update_add_htlc,onion_routing_packet,byte,1366", - "msgtype,update_fulfill_htlc,130", - "msgdata,update_fulfill_htlc,channel_id,channel_id,", - "msgdata,update_fulfill_htlc,id,u64,", - "msgdata,update_fulfill_htlc,payment_preimage,byte,32", - "msgtype,update_fail_htlc,131", - "msgdata,update_fail_htlc,channel_id,channel_id,", - "msgdata,update_fail_htlc,id,u64,", - "msgdata,update_fail_htlc,len,u16,", - "msgdata,update_fail_htlc,reason,byte,len", - "msgtype,update_fail_malformed_htlc,135", - "msgdata,update_fail_malformed_htlc,channel_id,channel_id,", - "msgdata,update_fail_malformed_htlc,id,u64,", - "msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256,", - "msgdata,update_fail_malformed_htlc,failure_code,u16,", - "msgtype,commitment_signed,132", - "msgdata,commitment_signed,channel_id,channel_id,", - "msgdata,commitment_signed,signature,signature,", - "msgdata,commitment_signed,num_htlcs,u16,", - "msgdata,commitment_signed,htlc_signature,signature,num_htlcs", - "msgtype,revoke_and_ack,133", - "msgdata,revoke_and_ack,channel_id,channel_id,", - "msgdata,revoke_and_ack,per_commitment_secret,byte,32", - "msgdata,revoke_and_ack,next_per_commitment_point,point,", - "msgtype,update_fee,134", - "msgdata,update_fee,channel_id,channel_id,", - "msgdata,update_fee,feerate_per_kw,u32,", - "msgtype,channel_reestablish,136", - "msgdata,channel_reestablish,channel_id,channel_id,", - "msgdata,channel_reestablish,next_commitment_number,u64,", - "msgdata,channel_reestablish,next_revocation_number,u64,", - "msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32", - "msgdata,channel_reestablish,my_current_per_commitment_point,point,", -] diff --git a/contrib/pyln-proto/pyln/proto/message/bolt4/Makefile b/contrib/pyln-proto/pyln/proto/message/bolt4/Makefile deleted file mode 100755 index 341681951..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt4/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/make - -SPECDIR := ../../../../../../../lightning-rfc - -csv.py: $(SPECDIR)/04-onion-routing.md Makefile - SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/ "\1",/'; echo ']') > $@ - chmod a+x $@ diff --git a/contrib/pyln-proto/pyln/proto/message/bolt4/bolt.py b/contrib/pyln-proto/pyln/proto/message/bolt4/bolt.py deleted file mode 100644 index 565c41228..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt4/bolt.py +++ /dev/null @@ -1,5 +0,0 @@ -from pyln.proto.message import MessageNamespace -from .csv import csv - - -namespace = MessageNamespace(csv_lines=csv) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt4/csv.py b/contrib/pyln-proto/pyln/proto/message/bolt4/csv.py deleted file mode 100755 index f51bcf829..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt4/csv.py +++ /dev/null @@ -1,55 +0,0 @@ -csv = [ - "tlvtype,tlv_payload,amt_to_forward,2", - "tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64,", - "tlvtype,tlv_payload,outgoing_cltv_value,4", - "tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32,", - "tlvtype,tlv_payload,short_channel_id,6", - "tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id,", - "tlvtype,tlv_payload,payment_data,8", - "tlvdata,tlv_payload,payment_data,payment_secret,byte,32", - "tlvdata,tlv_payload,payment_data,total_msat,tu64,", - "msgtype,invalid_realm,PERM|1", - "msgtype,temporary_node_failure,NODE|2", - "msgtype,permanent_node_failure,PERM|NODE|2", - "msgtype,required_node_feature_missing,PERM|NODE|3", - "msgtype,invalid_onion_version,BADONION|PERM|4", - "msgdata,invalid_onion_version,sha256_of_onion,sha256,", - "msgtype,invalid_onion_hmac,BADONION|PERM|5", - "msgdata,invalid_onion_hmac,sha256_of_onion,sha256,", - "msgtype,invalid_onion_key,BADONION|PERM|6", - "msgdata,invalid_onion_key,sha256_of_onion,sha256,", - "msgtype,temporary_channel_failure,UPDATE|7", - "msgdata,temporary_channel_failure,len,u16,", - "msgdata,temporary_channel_failure,channel_update,byte,len", - "msgtype,permanent_channel_failure,PERM|8", - "msgtype,required_channel_feature_missing,PERM|9", - "msgtype,unknown_next_peer,PERM|10", - "msgtype,amount_below_minimum,UPDATE|11", - "msgdata,amount_below_minimum,htlc_msat,u64,", - "msgdata,amount_below_minimum,len,u16,", - "msgdata,amount_below_minimum,channel_update,byte,len", - "msgtype,fee_insufficient,UPDATE|12", - "msgdata,fee_insufficient,htlc_msat,u64,", - "msgdata,fee_insufficient,len,u16,", - "msgdata,fee_insufficient,channel_update,byte,len", - "msgtype,incorrect_cltv_expiry,UPDATE|13", - "msgdata,incorrect_cltv_expiry,cltv_expiry,u32,", - "msgdata,incorrect_cltv_expiry,len,u16,", - "msgdata,incorrect_cltv_expiry,channel_update,byte,len", - "msgtype,expiry_too_soon,UPDATE|14", - "msgdata,expiry_too_soon,len,u16,", - "msgdata,expiry_too_soon,channel_update,byte,len", - "msgtype,incorrect_or_unknown_payment_details,PERM|15", - "msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64,", - "msgdata,incorrect_or_unknown_payment_details,height,u32,", - "msgtype,final_incorrect_cltv_expiry,18", - "msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32,", - "msgtype,final_incorrect_htlc_amount,19", - "msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,", - "msgtype,channel_disabled,UPDATE|20", - "msgtype,expiry_too_far,21", - "msgtype,invalid_onion_payload,PERM|22", - "msgdata,invalid_onion_payload,type,varint,", - "msgdata,invalid_onion_payload,offset,u16,", - "msgtype,mpp_timeout,23", -] diff --git a/contrib/pyln-proto/pyln/proto/message/bolt7/Makefile b/contrib/pyln-proto/pyln/proto/message/bolt7/Makefile deleted file mode 100755 index 13a7d6847..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt7/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -#! /usr/bin/make - -SPECDIR := ../../../../../../../lightning-rfc - -csv.py: $(SPECDIR)/07-routing-gossip.md Makefile - SPECNUM=`basename $< | sed 's/-.*//'`; (echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/ "\1",/'; echo ']') > $@ - chmod a+x $@ diff --git a/contrib/pyln-proto/pyln/proto/message/bolt7/__init__.py b/contrib/pyln-proto/pyln/proto/message/bolt7/__init__.py deleted file mode 100644 index 2ba3aceb6..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt7/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .csv import csv -from .bolt import namespace -import sys - -__version__ = '0.0.1' - -__all__ = [ - 'csv', - 'namespace', -] - -mod = sys.modules[__name__] -for d in namespace.subtypes, namespace.tlvtypes, namespace.messagetypes: - for name in d: - setattr(mod, name, d[name]) - __all__.append(name) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt7/bolt.py b/contrib/pyln-proto/pyln/proto/message/bolt7/bolt.py deleted file mode 100644 index 565c41228..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt7/bolt.py +++ /dev/null @@ -1,5 +0,0 @@ -from pyln.proto.message import MessageNamespace -from .csv import csv - - -namespace = MessageNamespace(csv_lines=csv) diff --git a/contrib/pyln-proto/pyln/proto/message/bolt7/csv.py b/contrib/pyln-proto/pyln/proto/message/bolt7/csv.py deleted file mode 100755 index 6c33c7b66..000000000 --- a/contrib/pyln-proto/pyln/proto/message/bolt7/csv.py +++ /dev/null @@ -1,83 +0,0 @@ -csv = [ - "msgtype,announcement_signatures,259", - "msgdata,announcement_signatures,channel_id,channel_id,", - "msgdata,announcement_signatures,short_channel_id,short_channel_id,", - "msgdata,announcement_signatures,node_signature,signature,", - "msgdata,announcement_signatures,bitcoin_signature,signature,", - "msgtype,channel_announcement,256", - "msgdata,channel_announcement,node_signature_1,signature,", - "msgdata,channel_announcement,node_signature_2,signature,", - "msgdata,channel_announcement,bitcoin_signature_1,signature,", - "msgdata,channel_announcement,bitcoin_signature_2,signature,", - "msgdata,channel_announcement,len,u16,", - "msgdata,channel_announcement,features,byte,len", - "msgdata,channel_announcement,chain_hash,chain_hash,", - "msgdata,channel_announcement,short_channel_id,short_channel_id,", - "msgdata,channel_announcement,node_id_1,point,", - "msgdata,channel_announcement,node_id_2,point,", - "msgdata,channel_announcement,bitcoin_key_1,point,", - "msgdata,channel_announcement,bitcoin_key_2,point,", - "msgtype,node_announcement,257", - "msgdata,node_announcement,signature,signature,", - "msgdata,node_announcement,flen,u16,", - "msgdata,node_announcement,features,byte,flen", - "msgdata,node_announcement,timestamp,u32,", - "msgdata,node_announcement,node_id,point,", - "msgdata,node_announcement,rgb_color,byte,3", - "msgdata,node_announcement,alias,byte,32", - "msgdata,node_announcement,addrlen,u16,", - "msgdata,node_announcement,addresses,byte,addrlen", - "msgtype,channel_update,258", - "msgdata,channel_update,signature,signature,", - "msgdata,channel_update,chain_hash,chain_hash,", - "msgdata,channel_update,short_channel_id,short_channel_id,", - "msgdata,channel_update,timestamp,u32,", - "msgdata,channel_update,message_flags,byte,", - "msgdata,channel_update,channel_flags,byte,", - "msgdata,channel_update,cltv_expiry_delta,u16,", - "msgdata,channel_update,htlc_minimum_msat,u64,", - "msgdata,channel_update,fee_base_msat,u32,", - "msgdata,channel_update,fee_proportional_millionths,u32,", - "msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max", - "msgtype,query_short_channel_ids,261,gossip_queries", - "msgdata,query_short_channel_ids,chain_hash,chain_hash,", - "msgdata,query_short_channel_ids,len,u16,", - "msgdata,query_short_channel_ids,encoded_short_ids,byte,len", - "msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,", - "tlvtype,query_short_channel_ids_tlvs,query_flags,1", - "tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8,", - "tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...", - "msgtype,reply_short_channel_ids_end,262,gossip_queries", - "msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,", - "msgdata,reply_short_channel_ids_end,full_information,byte,", - "msgtype,query_channel_range,263,gossip_queries", - "msgdata,query_channel_range,chain_hash,chain_hash,", - "msgdata,query_channel_range,first_blocknum,u32,", - "msgdata,query_channel_range,number_of_blocks,u32,", - "msgdata,query_channel_range,tlvs,query_channel_range_tlvs,", - "tlvtype,query_channel_range_tlvs,query_option,1", - "tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint,", - "msgtype,reply_channel_range,264,gossip_queries", - "msgdata,reply_channel_range,chain_hash,chain_hash,", - "msgdata,reply_channel_range,first_blocknum,u32,", - "msgdata,reply_channel_range,number_of_blocks,u32,", - "msgdata,reply_channel_range,full_information,byte,", - "msgdata,reply_channel_range,len,u16,", - "msgdata,reply_channel_range,encoded_short_ids,byte,len", - "msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,", - "tlvtype,reply_channel_range_tlvs,timestamps_tlv,1", - "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8,", - "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...", - "tlvtype,reply_channel_range_tlvs,checksums_tlv,3", - "tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...", - "subtype,channel_update_timestamps", - "subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,", - "subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,", - "subtype,channel_update_checksums", - "subtypedata,channel_update_checksums,checksum_node_id_1,u32,", - "subtypedata,channel_update_checksums,checksum_node_id_2,u32,", - "msgtype,gossip_timestamp_filter,265,gossip_queries", - "msgdata,gossip_timestamp_filter,chain_hash,chain_hash,", - "msgdata,gossip_timestamp_filter,first_timestamp,u32,", - "msgdata,gossip_timestamp_filter,timestamp_range,u32,", -] diff --git a/contrib/pyln-spec/Makefile b/contrib/pyln-spec/Makefile new file mode 100755 index 000000000..6a493dff9 --- /dev/null +++ b/contrib/pyln-spec/Makefile @@ -0,0 +1,47 @@ +#! /usr/bin/make + +SPECDIR := ../../../lightning-rfc +# This gives us something like 'v1.0-137-gae2d248b7ad8b0965f224c303019ba04c661008f' +GITDESCRIBE := $(shell git -C $(SPECDIR) describe --abbrev=40) +# PEP 440 requires numbers only, but allows -post (setuptools prefers .post though): +VERSION := $(shell echo $(GITDESCRIBE) | sed 's/^v//' | sed 's/-/.post/' | sed 's/-g.*//') +# This maintains -dirty, if present. +GITVERSION := $(shell echo $(GITDESCRIBE) | sed 's/.*-g//') + +BOLTS := 1 2 4 7 + +DIRS := $(foreach b,$(BOLTS),bolt$b) +CODE_DIRS := $(foreach b,$(BOLTS),bolt$b/pyln/spec/bolt$b) + +check: $(DIRS:%=check-pytest-%) +check-pytest-%: + cd $* && pytest + +check-source: check-source-flake8 check-source-mypy +check-source-flake8: $(DIRS:%=check-source-flake8-%) +check-source-mypy: $(DIRS:%=check-source-mypy-%) + +check-source-flake8-%: + cd $* && flake8 --ignore=E501,E731,W503 --exclude=gen.py + +# mypy . does not recurse. I have no idea why... +check-source-mypy-%: + cd $* && mypy --ignore-missing-imports `find * -name '*.py'` + +refresh: $(CODE_DIRS:%=%/gen_version.py) + +bolt1/pyln/spec/bolt1/gen.py: $(SPECDIR)/01-messaging.md Makefile +bolt2/pyln/spec/bolt2/gen.py: $(SPECDIR)/02-peer-protocol.md Makefile +bolt4/pyln/spec/bolt4/gen.py: $(SPECDIR)/04-onion-routing.md Makefile +bolt7/pyln/spec/bolt7/gen.py: $(SPECDIR)/07-routing-gossip.md Makefile + +%/gen_version.py: %/gen.py + echo '__version__ = "$(VERSION)"' > $@ + echo '__gitversion__ = "$(GITVERSION)"' >> $@ + +# We update iff it has changed. +$(CODE_DIRS:%=%/gen.py): + @(echo csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $< | sed 's/\(.*\)/ "\1",/'; echo ']') > $@.tmp + @echo 'desc = "'`head -n1 $< | cut -c3-`'"' >> $@.tmp + @(echo -n 'text = """'; sed 's,\\,\\\\,g' < $<; echo '"""') >> $@.tmp + @if cmp $@ $@.tmp >/dev/null 2>&1; then rm $@.tmp; else mv $@.tmp $@; fi diff --git a/contrib/pyln-proto/pyln/proto/message/bolt2/bolt.py b/contrib/pyln-spec/bolt.py similarity index 81% rename from contrib/pyln-proto/pyln/proto/message/bolt2/bolt.py rename to contrib/pyln-spec/bolt.py index 565c41228..dc675370a 100644 --- a/contrib/pyln-proto/pyln/proto/message/bolt2/bolt.py +++ b/contrib/pyln-spec/bolt.py @@ -1,5 +1,5 @@ from pyln.proto.message import MessageNamespace -from .csv import csv +from .gen import csv namespace = MessageNamespace(csv_lines=csv) diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/__init__.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/__init__.py new file mode 120000 index 000000000..52f300f8a --- /dev/null +++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/__init__.py @@ -0,0 +1 @@ +../../../../subinit.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/bolt.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/bolt.py new file mode 120000 index 000000000..c22b879fc --- /dev/null +++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/bolt.py @@ -0,0 +1 @@ +../../../../bolt.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen.py new file mode 100644 index 000000000..4bfcc2e33 --- /dev/null +++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen.py @@ -0,0 +1,976 @@ +csv = [ + "msgtype,init,16", + "msgdata,init,gflen,u16,", + "msgdata,init,globalfeatures,byte,gflen", + "msgdata,init,flen,u16,", + "msgdata,init,features,byte,flen", + "msgdata,init,tlvs,init_tlvs,", + "tlvtype,init_tlvs,networks,1", + "tlvdata,init_tlvs,networks,chains,chain_hash,...", + "msgtype,error,17", + "msgdata,error,channel_id,channel_id,", + "msgdata,error,len,u16,", + "msgdata,error,data,byte,len", + "msgtype,ping,18", + "msgdata,ping,num_pong_bytes,u16,", + "msgdata,ping,byteslen,u16,", + "msgdata,ping,ignored,byte,byteslen", + "msgtype,pong,19", + "msgdata,pong,byteslen,u16,", + "msgdata,pong,ignored,byte,byteslen", + "tlvtype,n1,tlv1,1", + "tlvdata,n1,tlv1,amount_msat,tu64,", + "tlvtype,n1,tlv2,2", + "tlvdata,n1,tlv2,scid,short_channel_id,", + "tlvtype,n1,tlv3,3", + "tlvdata,n1,tlv3,node_id,point,", + "tlvdata,n1,tlv3,amount_msat_1,u64,", + "tlvdata,n1,tlv3,amount_msat_2,u64,", + "tlvtype,n1,tlv4,254", + "tlvdata,n1,tlv4,cltv_delta,u16,", + "tlvtype,n2,tlv1,0", + "tlvdata,n2,tlv1,amount_msat,tu64,", + "tlvtype,n2,tlv2,11", + "tlvdata,n2,tlv2,cltv_expiry,tu32,", +] +desc = "BOLT #1: Base Protocol" +text = """# BOLT #1: Base Protocol + +## Overview + +This protocol assumes an underlying authenticated and ordered transport mechanism that takes care of framing individual messages. +[BOLT #8](08-transport.md) specifies the canonical transport layer used in Lightning, though it can be replaced by any transport that fulfills the above guarantees. + +The default TCP port is 9735. This corresponds to hexadecimal `0x2607`: the Unicode code point for LIGHTNING.[1](#reference-1) + +All data fields are unsigned big-endian unless otherwise specified. + +## Table of Contents + + * [Connection Handling and Multiplexing](#connection-handling-and-multiplexing) + * [Lightning Message Format](#lightning-message-format) + * [Type-Length-Value Format](#type-length-value-format) + * [Fundamental Types](#fundamental-types) + * [Setup Messages](#setup-messages) + * [The `init` Message](#the-init-message) + * [The `error` Message](#the-error-message) + * [Control Messages](#control-messages) + * [The `ping` and `pong` Messages](#the-ping-and-pong-messages) + * [Appendix A: BigSize Test Vectors](#appendix-a-bigsize-test-vectors) + * [Appendix B: Type-Length-Value Test Vectors](#appendix-b-type-length-value-test-vectors) + * [Appendix C: Message Extension](#appendix-c-message-extension) + * [Acknowledgments](#acknowledgments) + * [References](#references) + * [Authors](#authors) + +## Connection Handling and Multiplexing + +Implementations MUST use a single connection per peer; channel messages (which include a channel ID) are multiplexed over this single connection. + +## Lightning Message Format + +After decryption, all Lightning messages are of the form: + +1. `type`: a 2-byte big-endian field indicating the type of message +2. `payload`: a variable-length payload that comprises the remainder of + the message and that conforms to a format matching the `type` +3. `extension`: an optional [TLV stream](#type-length-value-format) + +The `type` field indicates how to interpret the `payload` field. +The format for each individual type is defined by a specification in this repository. +The type follows the _it's ok to be odd_ rule, so nodes MAY send _odd_-numbered types without ascertaining that the recipient understands it. + +The messages are grouped logically into five groups, ordered by the most significant bit that is set: + + - Setup & Control (types `0`-`31`): messages related to connection setup, control, supported features, and error reporting (described below) + - Channel (types `32`-`127`): messages used to setup and tear down micropayment channels (described in [BOLT #2](02-peer-protocol.md)) + - Commitment (types `128`-`255`): messages related to updating the current commitment transaction, which includes adding, revoking, and settling HTLCs as well as updating fees and exchanging signatures (described in [BOLT #2](02-peer-protocol.md)) + - Routing (types `256`-`511`): messages containing node and channel announcements, as well as any active route exploration (described in [BOLT #7](07-routing-gossip.md)) + - Custom (types `32768`-`65535`): experimental and application-specific messages + +The size of the message is required by the transport layer to fit into a 2-byte unsigned int; therefore, the maximum possible size is 65535 bytes. + +A sending node: + - MUST NOT send an evenly-typed message not listed here without prior negotiation. + - MUST NOT send evenly-typed TLV records in the `extension` without prior negotiation. + - that negotiates an option in this specification: + - MUST include all the fields annotated with that option. + - When defining custom messages: + - SHOULD pick a random `type` to avoid collision with other custom types. + - SHOULD pick a `type` that doesn't conflict with other experiments listed in [this issue](https://github.com/lightningnetwork/lightning-rfc/issues/716). + - SHOULD pick an odd `type` identifiers when regular nodes should ignore the + additional data. + - SHOULD pick an even `type` identifiers when regular nodes should reject + the message and close the connection. + +A receiving node: + - upon receiving a message of _odd_, unknown type: + - MUST ignore the received message. + - upon receiving a message of _even_, unknown type: + - MUST close the connection. + - MAY fail the channels. + - upon receiving a known message with insufficient length for the contents: + - MUST close the connection. + - MAY fail the channels. + - upon receiving a message with an `extension`: + - MAY ignore the `extension`. + - Otherwise, if the `extension` is invalid: + - MUST close the connection. + - MAY fail the channels. + +### Rationale + +By default `SHA2` and Bitcoin public keys are both encoded as +big endian, thus it would be unusual to use a different endian for +other fields. + +Length is limited to 65535 bytes by the cryptographic wrapping, and +messages in the protocol are never more than that length anyway. + +The _it's ok to be odd_ rule allows for future optional extensions +without negotiation or special coding in clients. The _extension_ field +similarly allows for future expansion by letting senders include additional +TLV data. Note that an _extension_ field can only be added when the message +`payload` doesn't already fill the 65535 bytes maximum length. + +Implementations may prefer to have message data aligned on an 8-byte +boundary (the largest natural alignment requirement of any type here); +however, adding a 6-byte padding after the type field was considered +wasteful: alignment may be achieved by decrypting the message into +a buffer with 6-bytes of pre-padding. + +## Type-Length-Value Format + +Throughout the protocol, a TLV (Type-Length-Value) format is used to allow for +the backwards-compatible addition of new fields to existing message types. + +A `tlv_record` represents a single field, encoded in the form: + +* [`bigsize`: `type`] +* [`bigsize`: `length`] +* [`length`: `value`] + +A `tlv_stream` is a series of (possibly zero) `tlv_record`s, represented as the +concatenation of the encoded `tlv_record`s. When used to extend existing +messages, a `tlv_stream` is typically placed after all currently defined fields. + +The `type` is encoded using the BigSize format. It functions as a +message-specific, 64-bit identifier for the `tlv_record` determining how the +contents of `value` should be decoded. `type` identifiers below 2^16 are +reserved for use in this specification. `type` identifiers greater than or equal +to 2^16 are available for custom records. Any record not defined in this +specification is considered a custom record. This includes experimental and +application-specific messages. + +The `length` is encoded using the BigSize format signaling the size of +`value` in bytes. + +The `value` depends entirely on the `type`, and should be encoded or decoded +according to the message-specific format determined by `type`. + +### Requirements + +The sending node: + - MUST order `tlv_record`s in a `tlv_stream` by monotonically-increasing `type`. + - MUST minimally encode `type` and `length`. + - When defining custom record `type` identifiers: + - SHOULD pick random `type` identifiers to avoid collision with other + custom types. + - SHOULD pick odd `type` identifiers when regular nodes should ignore the + additional data. + - SHOULD pick even `type` identifiers when regular nodes should reject the + full tlv stream containing the custom record. + - SHOULD NOT use redundant, variable-length encodings in a `tlv_record`. + +The receiving node: + - if zero bytes remain before parsing a `type`: + - MUST stop parsing the `tlv_stream`. + - if a `type` or `length` is not minimally encoded: + - MUST fail to parse the `tlv_stream`. + - if decoded `type`s are not monotonically-increasing: + - MUST fail to parse the `tlv_stream`. + - if `length` exceeds the number of bytes remaining in the message: + - MUST fail to parse the `tlv_stream`. + - if `type` is known: + - MUST decode the next `length` bytes using the known encoding for `type`. + - if `length` is not exactly equal to that required for the known encoding for `type`: + - MUST fail to parse the `tlv_stream`. + - if variable-length fields within the known encoding for `type` are not minimal: + - MUST fail to parse the `tlv_stream`. + - otherwise, if `type` is unknown: + - if `type` is even: + - MUST fail to parse the `tlv_stream`. + - otherwise, if `type` is odd: + - MUST discard the next `length` bytes. + +### Rationale + +The primary advantage in using TLV is that a reader is able to ignore new fields +that it does not understand, since each field carries the exact size of the +encoded element. Without TLV, even if a node does not wish to use a particular +field, the node is forced to add parsing logic for that field in order to +determine the offset of any fields that follow. + +The monotonicity constraint ensures that all `type`s are unique and can appear +at most once. Fields that map to complex objects, e.g. vectors, maps, or +structs, should do so by defining the encoding such that the object is +serialized within a single `tlv_record`. The uniqueness constraint, among other +things, enables the following optimizations: + - canonical ordering is defined independent of the encoded `value`s. + - canonical ordering can be known at compile-time, rather than being determined + dynamically at the time of encoding. + - verifying canonical ordering requires less state and is less-expensive. + - variable-size fields can reserve their expected size up front, rather than + appending elements sequentially and incurring double-and-copy overhead. + +The use of a bigsize for `type` and `length` permits a space savings for small +`type`s or short `value`s. This potentially leaves more space for application +data over the wire or in an onion payload. + +All `type`s must appear in increasing order to create a canonical encoding of +the underlying `tlv_record`s. This is crucial when computing signatures over a +`tlv_stream`, as it ensures verifiers will be able to recompute the same message +digest as the signer. Note that the canonical ordering over the set of fields +can be enforced even if the verifier does not understand what the fields +contain. + +Writers should avoid using redundant, variable-length encodings in a +`tlv_record` since this results in encoding the length twice and complicates +computing the outer length. As an example, when writing a variable length byte +array, the `value` should contain only the raw bytes and forgo an additional +internal length since the `tlv_record` already carries the number of bytes that +follow. On the other hand, if a `tlv_record` contains multiple, variable-length +elements then this would not be considered redundant, and is needed to allow the +receiver to parse individual elements from `value`. + +## Fundamental Types + +Various fundamental types are referred to in the message specifications: + +* `byte`: an 8-bit byte +* `u16`: a 2 byte unsigned integer +* `u32`: a 4 byte unsigned integer +* `u64`: an 8 byte unsigned integer + +Inside TLV records which contain a single value, leading zeros in +integers can be omitted: + +* `tu16`: a 0 to 2 byte unsigned integer +* `tu32`: a 0 to 4 byte unsigned integer +* `tu64`: a 0 to 8 byte unsigned integer + +The following convenience types are also defined: + +* `chain_hash`: a 32-byte chain identifier (see [BOLT #0](00-introduction.md#glossary-and-terminology-guide)) +* `channel_id`: a 32-byte channel_id (see [BOLT #2](02-peer-protocol.md#definition-of-channel-id)) +* `sha256`: a 32-byte SHA2-256 hash +* `signature`: a 64-byte bitcoin Elliptic Curve signature +* `point`: a 33-byte Elliptic Curve point (compressed encoding as per [SEC 1 standard](http://www.secg.org/sec1-v2.pdf#subsubsection.2.3.3)) +* `short_channel_id`: an 8 byte value identifying a channel (see [BOLT #7](07-routing-gossip.md#definition-of-short-channel-id)) +* `bigsize`: a variable-length, unsigned integer similar to Bitcoin's CompactSize encoding, but big-endian. Described in [BigSize](#appendix-a-bigsize-test-vectors). + +## Setup Messages + +### The `init` Message + +Once authentication is complete, the first message reveals the features supported or required by this node, even if this is a reconnection. + +[BOLT #9](09-features.md) specifies lists of features. Each feature is generally represented by 2 bits. The least-significant bit is numbered 0, which is _even_, and the next most significant bit is numbered 1, which is _odd_. For historical reasons, features are divided into global and local feature bitmasks. + +The `features` field MUST be padded to bytes with 0s. + +1. type: 16 (`init`) +2. data: + * [`u16`:`gflen`] + * [`gflen*byte`:`globalfeatures`] + * [`u16`:`flen`] + * [`flen*byte`:`features`] + * [`init_tlvs`:`tlvs`] + +1. tlvs: `init_tlvs` +2. types: + 1. type: 1 (`networks`) + 2. data: + * [`...*chain_hash`:`chains`] + + +The optional `networks` indicates the chains the node is interested in. + +#### Requirements + +The sending node: + - MUST send `init` as the first Lightning message for any connection. + - MUST set feature bits as defined in [BOLT #9](09-features.md). + - MUST set any undefined feature bits to 0. + - SHOULD NOT set features greater than 13 in `globalfeatures`. + - SHOULD use the minimum length required to represent the `features` field. + - SHOULD set `networks` to all chains it will gossip or open channels for. + +The receiving node: + - MUST wait to receive `init` before sending any other messages. + - MUST combine (logical OR) the two feature bitmaps into one logical `features` map. + - MUST respond to known feature bits as specified in [BOLT #9](09-features.md). + - upon receiving unknown _odd_ feature bits that are non-zero: + - MUST ignore the bit. + - upon receiving unknown _even_ feature bits that are non-zero: + - MUST fail the connection. + - upon receiving `networks` containing no common chains + - MAY fail the connection. + - if the feature vector does not set all known, transitive dependencies: + - MUST fail the connection. + +#### Rationale + +There used to be two feature bitfields here, but for backwards compatibility they're now +combined into one. + +This semantic allows both future incompatible changes and future backward compatible changes. Bits should generally be assigned in pairs, in order that optional features may later become compulsory. + +Nodes wait for receipt of the other's features to simplify error +diagnosis when features are incompatible. + +Since all networks share the same port, but most implementations only +support a single network, the `networks` fields avoids nodes +erroneously believing they will receive updates about their preferred +network, or that they can open channels. + +### The `error` Message + +For simplicity of diagnosis, it's often useful to tell a peer that something is incorrect. + +1. type: 17 (`error`) +2. data: + * [`channel_id`:`channel_id`] + * [`u16`:`len`] + * [`len*byte`:`data`] + +The 2-byte `len` field indicates the number of bytes in the immediately following field. + +#### Requirements + +The channel is referred to by `channel_id`, unless `channel_id` is 0 (i.e. all bytes are 0), in which case it refers to all channels. + +The funding node: + - for all error messages sent before (and including) the `funding_created` message: + - MUST use `temporary_channel_id` in lieu of `channel_id`. + +The fundee node: + - for all error messages sent before (and not including) the `funding_signed` message: + - MUST use `temporary_channel_id` in lieu of `channel_id`. + +A sending node: + - when sending `error`: + - MUST fail the channel referred to by the error message. + - SHOULD send `error` for protocol violations or internal errors that make channels unusable or that make further communication unusable. + - SHOULD send `error` with the unknown `channel_id` in reply to messages of type `32`-`255` related to unknown channels. + - MAY send an empty `data` field. + - when failure was caused by an invalid signature check: + - SHOULD include the raw, hex-encoded transaction in reply to a `funding_created`, `funding_signed`, `closing_signed`, or `commitment_signed` message. + - when `channel_id` is 0: + - MUST fail all channels with the receiving node. + - MUST close the connection. + - MUST set `len` equal to the length of `data`. + +The receiving node: + - upon receiving `error`: + - MUST fail the channel referred to by the error message, if that channel is with the sending node. + - if no existing channel is referred to by the message: + - MUST ignore the message. + - MUST truncate `len` to the remainder of the packet (if it's larger). + - if `data` is not composed solely of printable ASCII characters (For reference: the printable character set includes byte values 32 through 126, inclusive): + - SHOULD NOT print out `data` verbatim. + +#### Rationale + +There are unrecoverable errors that require an abort of conversations; +if the connection is simply dropped, then the peer may retry the +connection. It's also useful to describe protocol violations for +diagnosis, as this indicates that one peer has a bug. + +It may be wise not to distinguish errors in production settings, lest +it leak information — hence, the optional `data` field. + +## Control Messages + +### The `ping` and `pong` Messages + +In order to allow for the existence of long-lived TCP connections, at +times it may be required that both ends keep alive the TCP connection at the +application level. Such messages also allow obfuscation of traffic patterns. + +1. type: 18 (`ping`) +2. data: + * [`u16`:`num_pong_bytes`] + * [`u16`:`byteslen`] + * [`byteslen*byte`:`ignored`] + +The `pong` message is to be sent whenever a `ping` message is received. It +serves as a reply and also serves to keep the connection alive, while +explicitly notifying the other end that the receiver is still active. Within +the received `ping` message, the sender will specify the number of bytes to be +included within the data payload of the `pong` message. + +1. type: 19 (`pong`) +2. data: + * [`u16`:`byteslen`] + * [`byteslen*byte`:`ignored`] + +#### Requirements + +A node sending a `ping` message: + - SHOULD set `ignored` to 0s. + - MUST NOT set `ignored` to sensitive data such as secrets or portions of initialized +memory. + - if it doesn't receive a corresponding `pong`: + - MAY terminate the network connection, + - and MUST NOT fail the channels in this case. + - SHOULD NOT send `ping` messages more often than once every 30 seconds. + +A node sending a `pong` message: + - SHOULD set `ignored` to 0s. + - MUST NOT set `ignored` to sensitive data such as secrets or portions of initialized + memory. + +A node receiving a `ping` message: + - SHOULD fail the channels if it has received significantly in excess of one `ping` per 30 seconds. + - if `num_pong_bytes` is less than 65532: + - MUST respond by sending a `pong` message, with `byteslen` equal to `num_pong_bytes`. + - otherwise (`num_pong_bytes` is **not** less than 65532): + - MUST ignore the `ping`. + +A node receiving a `pong` message: + - if `byteslen` does not correspond to any `ping`'s `num_pong_bytes` value it has sent: + - MAY fail the channels. + +### Rationale + +The largest possible message is 65535 bytes; thus, the maximum sensible `byteslen` +is 65531 — in order to account for the type field (`pong`) and the `byteslen` itself. This allows +a convenient cutoff for `num_pong_bytes` to indicate that no reply should be sent. + +Connections between nodes within the network may be long lived, as payment +channels have an indefinite lifetime. However, it's likely that +no new data will be +exchanged for a +significant portion of a connection's lifetime. Also, on several platforms it's possible that Lightning +clients will be put to sleep without prior warning. Hence, a +distinct `ping` message is used, in order to probe for the liveness of the connection on +the other side, as well as to keep the established connection active. + +Additionally, the ability for a sender to request that the receiver send a +response with a particular number of bytes enables nodes on the network to +create _synthetic_ traffic. Such traffic can be used to partially defend +against packet and timing analysis — as nodes can fake the traffic patterns of +typical exchanges without applying any true updates to their respective +channels. + +When combined with the onion routing protocol defined in +[BOLT #4](04-onion-routing.md), +careful statistically driven synthetic traffic can serve to further bolster the +privacy of participants within the network. + +Limited precautions are recommended against `ping` flooding, however some +latitude is given because of network delays. Note that there are other methods +of incoming traffic flooding (e.g. sending _odd_ unknown message types, or padding +every message maximally). + +Finally, the usage of periodic `ping` messages serves to promote frequent key +rotations as specified within [BOLT #8](08-transport.md). + +## Appendix A: BigSize Test Vectors + +The following test vectors can be used to assert the correctness of a BigSize +implementation used in the TLV format. The format is identical to the +CompactSize encoding used in bitcoin, but replaces the little-endian encoding of +multi-byte values with big-endian. + +Values encoded with BigSize will produce an encoding of either 1, 3, 5, or 9 +bytes depending on the size of the integer. The encoding is a piece-wise +function that takes a `uint64` value `x` and produces: +``` + uint8(x) if x < 0xfd + 0xfd + be16(uint16(x)) if x < 0x10000 + 0xfe + be32(uint32(x)) if x < 0x100000000 + 0xff + be64(x) otherwise. +``` + +Here `+` denotes concatenation and `be16`, `be32`, and `be64` produce a +big-endian encoding of the input for 16, 32, and 64-bit integers, respectively. + +A value is said to be _minimally encoded_ if it could not be encoded using +fewer bytes. For example, a BigSize encoding that occupies 5 bytes +but whose value is less than 0x10000 is not minimally encoded. All values +decoded with BigSize should be checked to ensure they are minimally encoded. + +### BigSize Decoding Tests + +The following is an example of how to execute the BigSize decoding tests. +```golang +func testReadBigSize(t *testing.T, test bigSizeTest) { + var buf [8]byte + r := bytes.NewReader(test.Bytes) + val, err := tlv.ReadBigSize(r, &buf) + if err != nil && err.Error() != test.ExpErr { + t.Fatalf("expected decoding error: %v, got: %v", + test.ExpErr, err) + } + + // If we expected a decoding error, there's no point checking the value. + if test.ExpErr != "" { + return + } + + if val != test.Value { + t.Fatalf("expected value: %d, got %d", test.Value, val) + } +} +``` + +A correct implementation should pass against these test vectors: +```json +[ + { + "name": "zero", + "value": 0, + "bytes": "00" + }, + { + "name": "one byte high", + "value": 252, + "bytes": "fc" + }, + { + "name": "two byte low", + "value": 253, + "bytes": "fd00fd" + }, + { + "name": "two byte high", + "value": 65535, + "bytes": "fdffff" + }, + { + "name": "four byte low", + "value": 65536, + "bytes": "fe00010000" + }, + { + "name": "four byte high", + "value": 4294967295, + "bytes": "feffffffff" + }, + { + "name": "eight byte low", + "value": 4294967296, + "bytes": "ff0000000100000000" + }, + { + "name": "eight byte high", + "value": 18446744073709551615, + "bytes": "ffffffffffffffffff" + }, + { + "name": "two byte not canonical", + "value": 0, + "bytes": "fd00fc", + "exp_error": "decoded bigsize is not canonical" + }, + { + "name": "four byte not canonical", + "value": 0, + "bytes": "fe0000ffff", + "exp_error": "decoded bigsize is not canonical" + }, + { + "name": "eight byte not canonical", + "value": 0, + "bytes": "ff00000000ffffffff", + "exp_error": "decoded bigsize is not canonical" + }, + { + "name": "two byte short read", + "value": 0, + "bytes": "fd00", + "exp_error": "unexpected EOF" + }, + { + "name": "four byte short read", + "value": 0, + "bytes": "feffff", + "exp_error": "unexpected EOF" + }, + { + "name": "eight byte short read", + "value": 0, + "bytes": "ffffffffff", + "exp_error": "unexpected EOF" + }, + { + "name": "one byte no read", + "value": 0, + "bytes": "", + "exp_error": "EOF" + }, + { + "name": "two byte no read", + "value": 0, + "bytes": "fd", + "exp_error": "unexpected EOF" + }, + { + "name": "four byte no read", + "value": 0, + "bytes": "fe", + "exp_error": "unexpected EOF" + }, + { + "name": "eight byte no read", + "value": 0, + "bytes": "ff", + "exp_error": "unexpected EOF" + } +] +``` + +### BigSize Encoding Tests + +The following is an example of how to execute the BigSize encoding tests. +```golang +func testWriteBigSize(t *testing.T, test bigSizeTest) { + var ( + w bytes.Buffer + buf [8]byte + ) + err := tlv.WriteBigSize(&w, test.Value, &buf) + if err != nil { + t.Fatalf("unable to encode %d as bigsize: %v", + test.Value, err) + } + + if bytes.Compare(w.Bytes(), test.Bytes) != 0 { + t.Fatalf("expected bytes: %v, got %v", + test.Bytes, w.Bytes()) + } +} +``` + +A correct implementation should pass against the following test vectors: +```json +[ + { + "name": "zero", + "value": 0, + "bytes": "00" + }, + { + "name": "one byte high", + "value": 252, + "bytes": "fc" + }, + { + "name": "two byte low", + "value": 253, + "bytes": "fd00fd" + }, + { + "name": "two byte high", + "value": 65535, + "bytes": "fdffff" + }, + { + "name": "four byte low", + "value": 65536, + "bytes": "fe00010000" + }, + { + "name": "four byte high", + "value": 4294967295, + "bytes": "feffffffff" + }, + { + "name": "eight byte low", + "value": 4294967296, + "bytes": "ff0000000100000000" + }, + { + "name": "eight byte high", + "value": 18446744073709551615, + "bytes": "ffffffffffffffffff" + } +] +``` + +## Appendix B: Type-Length-Value Test Vectors + +The following tests assume that two separate TLV namespaces exist: n1 and n2. + +The n1 namespace supports the following TLV types: + +1. tlvs: `n1` +2. types: + 1. type: 1 (`tlv1`) + 2. data: + * [`tu64`:`amount_msat`] + 1. type: 2 (`tlv2`) + 2. data: + * [`short_channel_id`:`scid`] + 1. type: 3 (`tlv3`) + 2. data: + * [`point`:`node_id`] + * [`u64`:`amount_msat_1`] + * [`u64`:`amount_msat_2`] + 1. type: 254 (`tlv4`) + 2. data: + * [`u16`:`cltv_delta`] + +The n2 namespace supports the following TLV types: + +1. tlvs: `n2` +2. types: + 1. type: 0 (`tlv1`) + 2. data: + * [`tu64`:`amount_msat`] + 1. type: 11 (`tlv2`) + 2. data: + * [`tu32`:`cltv_expiry`] + +### TLV Decoding Failures + +The following TLV streams in any namespace should trigger a decoding failure: + +1. Invalid stream: 0xfd +2. Reason: type truncated + +1. Invalid stream: 0xfd01 +2. Reason: type truncated + +1. Invalid stream: 0xfd0001 00 +2. Reason: not minimally encoded type + +1. Invalid stream: 0xfd0101 +2. Reason: missing length + +1. Invalid stream: 0x0f fd +2. Reason: (length truncated) + +1. Invalid stream: 0x0f fd26 +2. Reason: (length truncated) + +1. Invalid stream: 0x0f fd2602 +2. Reason: missing value + +1. Invalid stream: 0x0f fd0001 00 +2. Reason: not minimally encoded length + +1. Invalid stream: 0x0f fd0201 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 +2. Reason: value truncated + +The following TLV streams in either namespace should trigger a +decoding failure: + +1. Invalid stream: 0x12 00 +2. Reason: unknown even type. + +1. Invalid stream: 0xfd0102 00 +2. Reason: unknown even type. + +1. Invalid stream: 0xfe01000002 00 +2. Reason: unknown even type. + +1. Invalid stream: 0xff0100000000000002 00 +2. Reason: unknown even type. + +The following TLV streams in namespace `n1` should trigger a decoding +failure: + +1. Invalid stream: 0x01 09 ffffffffffffffffff +2. Reason: greater than encoding length for `n1`s `tlv1`. + +1. Invalid stream: 0x01 01 00 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 02 0001 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 03 000100 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 04 00010000 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 05 0001000000 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 06 000100000000 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 07 00010000000000 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x01 08 0001000000000000 +2. Reason: encoding for `n1`s `tlv1`s `amount_msat` is not minimal + +1. Invalid stream: 0x02 07 01010101010101 +2. Reason: less than encoding length for `n1`s `tlv2`. + +1. Invalid stream: 0x02 09 010101010101010101 +2. Reason: greater than encoding length for `n1`s `tlv2`. + +1. Invalid stream: 0x03 21 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb +2. Reason: less than encoding length for `n1`s `tlv3`. + +1. Invalid stream: 0x03 29 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb0000000000000001 +2. Reason: less than encoding length for `n1`s `tlv3`. + +1. Invalid stream: 0x03 30 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb000000000000000100000000000001 +2. Reason: less than encoding length for `n1`s `tlv3`. + +1. Invalid stream: 0x03 31 043da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb00000000000000010000000000000002 +2. Reason: `n1`s `node_id` is not a valid point. + +1. Invalid stream: 0x03 32 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb0000000000000001000000000000000001 +2. Reason: greater than encoding length for `n1`s `tlv3`. + +1. Invalid stream: 0xfd00fe 00 +2. Reason: less than encoding length for `n1`s `tlv4`. + +1. Invalid stream: 0xfd00fe 01 01 +2. Reason: less than encoding length for `n1`s `tlv4`. + +1. Invalid stream: 0xfd00fe 03 010101 +2. Reason: greater than encoding length for `n1`s `tlv4`. + +1. Invalid stream: 0x00 00 +2. Reason: unknown even field for `n1`s namespace. + +### TLV Decoding Successes + +The following TLV streams in either namespace should correctly decode, +and be ignored: + +1. Valid stream: 0x +2. Explanation: empty message + +1. Valid stream: 0x21 00 +2. Explanation: Unknown odd type. + +1. Valid stream: 0xfd0201 00 +2. Explanation: Unknown odd type. + +1. Valid stream: 0xfd00fd 00 +2. Explanation: Unknown odd type. + +1. Valid stream: 0xfd00ff 00 +2. Explanation: Unknown odd type. + +1. Valid stream: 0xfe02000001 00 +2. Explanation: Unknown odd type. + +1. Valid stream: 0xff0200000000000001 00 +2. Explanation: Unknown odd type. + +The following TLV streams in `n1` namespace should correctly decode, +with the values given here: + +1. Valid stream: 0x01 00 +2. Values: `tlv1` `amount_msat`=0 + +1. Valid stream: 0x01 01 01 +2. Values: `tlv1` `amount_msat`=1 + +1. Valid stream: 0x01 02 0100 +2. Values: `tlv1` `amount_msat`=256 + +1. Valid stream: 0x01 03 010000 +2. Values: `tlv1` `amount_msat`=65536 + +1. Valid stream: 0x01 04 01000000 +2. Values: `tlv1` `amount_msat`=16777216 + +1. Valid stream: 0x01 05 0100000000 +2. Values: `tlv1` `amount_msat`=4294967296 + +1. Valid stream: 0x01 06 010000000000 +2. Values: `tlv1` `amount_msat`=1099511627776 + +1. Valid stream: 0x01 07 01000000000000 +2. Values: `tlv1` `amount_msat`=281474976710656 + +1. Valid stream: 0x01 08 0100000000000000 +2. Values: `tlv1` `amount_msat`=72057594037927936 + +1. Valid stream: 0x02 08 0000000000000226 +2. Values: `tlv2` `scid`=0x0x550 + +1. Valid stream: 0x03 31 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb00000000000000010000000000000002 +2. Values: `tlv3` `node_id`=023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb `amount_msat_1`=1 `amount_msat_2`=2 + +1. Valid stream: 0xfd00fe 02 0226 +2. Values: `tlv4` `cltv_delta`=550 + +### TLV Stream Decoding Failure + +Any appending of an invalid stream to a valid stream should trigger +a decoding failure. + +Any appending of a higher-numbered valid stream to a lower-numbered +valid stream should not trigger a decoding failure. + +In addition, the following TLV streams in namespace `n1` should +trigger a decoding failure: + +1. Invalid stream: 0x02 08 0000000000000226 01 01 2a +2. Reason: valid TLV records but invalid ordering + +1. Invalid stream: 0x02 08 0000000000000231 02 08 0000000000000451 +2. Reason: duplicate TLV type + +1. Invalid stream: 0x1f 00 0f 01 2a +2. Reason: valid (ignored) TLV records but invalid ordering + +1. Invalid stream: 0x1f 00 1f 01 2a +2. Reason: duplicate TLV type (ignored) + +The following TLV stream in namespace `n2` should trigger a decoding +failure: + +1. Invalid stream: 0xffffffffffffffffff 00 00 00 +2. Reason: valid TLV records but invalid ordering + +## Appendix C: Message Extension + +This section contains examples of valid and invalid extensions on the `init` +message. The base `init` message (without extensions) for these examples is +`0x001000000000` (all features turned off). + +The following `init` messages are valid: + +- `0x001000000000`: no extension provided +- `0x00100000000001012a030104`: the extension contains two _odd_ TLV records (with types `0x01` and `0x03`) + +The following `init` messages are invalid: + +- `0x00100000000001`: the extension is present but truncated +- `0x00100000000002012a`: the extension contains unknown _even_ TLV records (assuming that TLV type `0x02` is unknown) +- `0x001000000000010101010102`: the extension TLV stream is invalid (duplicate TLV record type `0x01`) + +Note that when messages are signed, the _extension_ is part of the signed bytes. +Nodes should store the _extension_ bytes even if they don't understand them to +be able to correctly verify signatures. + +## Acknowledgments + +[ TODO: (roasbeef); fin ] + +## References + +1. http://www.unicode.org/charts/PDF/U2600.pdf + +## Authors + +[ FIXME: Insert Author List ] + +![Creative Commons License](https://i.creativecommons.org/l/by/4.0/88x31.png "License CC-BY") +
+This work is licensed under a [Creative Commons Attribution 4.0 International License](http://creativecommons.org/licenses/by/4.0/). +""" diff --git a/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py new file mode 100644 index 000000000..4f6bc8b19 --- /dev/null +++ b/contrib/pyln-spec/bolt1/pyln/spec/bolt1/gen_version.py @@ -0,0 +1,2 @@ +__version__ = "1.0.post137" +__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56" diff --git a/contrib/pyln-spec/bolt1/requirements.txt b/contrib/pyln-spec/bolt1/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/contrib/pyln-spec/bolt1/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/contrib/pyln-spec/bolt1/setup.py b/contrib/pyln-spec/bolt1/setup.py new file mode 100644 index 000000000..2105b1153 --- /dev/null +++ b/contrib/pyln-spec/bolt1/setup.py @@ -0,0 +1,23 @@ +from pyln.spec.bolt1 import __version__, desc +from setuptools import setup +import io + +with io.open('requirements.txt', encoding='utf-8') as f: + requirements = [r for r in f.read().split('\n') if len(r)] + + +def do_setup(boltnum: int, version: str, desc: str): + setup(name='pyln-bolt{}'.format(boltnum), + version=version, + description=desc, + url='http://github.com/ElementsProject/lightning', + author='Rusty Russell', + author_email='rusty@rustcorp.com.au', + license='MIT', + packages=['pyln.spec.bolt{}'.format(boltnum)], + scripts=[], + zip_safe=True, + install_requires=requirements) + + +do_setup(1, __version__, desc) diff --git a/contrib/pyln-proto/tests/test_bolt1.py b/contrib/pyln-spec/bolt1/tests/test_bolt1.py similarity index 98% rename from contrib/pyln-proto/tests/test_bolt1.py rename to contrib/pyln-spec/bolt1/tests/test_bolt1.py index d64578dee..68e4d19e9 100644 --- a/contrib/pyln-proto/tests/test_bolt1.py +++ b/contrib/pyln-spec/bolt1/tests/test_bolt1.py @@ -1,6 +1,6 @@ #! /usr/bin/python3 from pyln.proto.message import Message, MessageNamespace -import pyln.proto.message.bolt1 as bolt1 +import pyln.spec.bolt1 as bolt1 import io diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/__init__.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/__init__.py new file mode 120000 index 000000000..52f300f8a --- /dev/null +++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/__init__.py @@ -0,0 +1 @@ +../../../../subinit.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/bolt.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/bolt.py new file mode 120000 index 000000000..c22b879fc --- /dev/null +++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/bolt.py @@ -0,0 +1 @@ +../../../../bolt.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen.py new file mode 100644 index 000000000..7b9f1857c --- /dev/null +++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen.py @@ -0,0 +1,1471 @@ +csv = [ + "msgtype,open_channel,32", + "msgdata,open_channel,chain_hash,chain_hash,", + "msgdata,open_channel,temporary_channel_id,byte,32", + "msgdata,open_channel,funding_satoshis,u64,", + "msgdata,open_channel,push_msat,u64,", + "msgdata,open_channel,dust_limit_satoshis,u64,", + "msgdata,open_channel,max_htlc_value_in_flight_msat,u64,", + "msgdata,open_channel,channel_reserve_satoshis,u64,", + "msgdata,open_channel,htlc_minimum_msat,u64,", + "msgdata,open_channel,feerate_per_kw,u32,", + "msgdata,open_channel,to_self_delay,u16,", + "msgdata,open_channel,max_accepted_htlcs,u16,", + "msgdata,open_channel,funding_pubkey,point,", + "msgdata,open_channel,revocation_basepoint,point,", + "msgdata,open_channel,payment_basepoint,point,", + "msgdata,open_channel,delayed_payment_basepoint,point,", + "msgdata,open_channel,htlc_basepoint,point,", + "msgdata,open_channel,first_per_commitment_point,point,", + "msgdata,open_channel,channel_flags,byte,", + "msgdata,open_channel,tlvs,open_channel_tlvs,", + "tlvtype,open_channel_tlvs,upfront_shutdown_script,0", + "tlvdata,open_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", + "msgtype,accept_channel,33", + "msgdata,accept_channel,temporary_channel_id,byte,32", + "msgdata,accept_channel,dust_limit_satoshis,u64,", + "msgdata,accept_channel,max_htlc_value_in_flight_msat,u64,", + "msgdata,accept_channel,channel_reserve_satoshis,u64,", + "msgdata,accept_channel,htlc_minimum_msat,u64,", + "msgdata,accept_channel,minimum_depth,u32,", + "msgdata,accept_channel,to_self_delay,u16,", + "msgdata,accept_channel,max_accepted_htlcs,u16,", + "msgdata,accept_channel,funding_pubkey,point,", + "msgdata,accept_channel,revocation_basepoint,point,", + "msgdata,accept_channel,payment_basepoint,point,", + "msgdata,accept_channel,delayed_payment_basepoint,point,", + "msgdata,accept_channel,htlc_basepoint,point,", + "msgdata,accept_channel,first_per_commitment_point,point,", + "msgdata,accept_channel,tlvs,accept_channel_tlvs,", + "tlvtype,accept_channel_tlvs,upfront_shutdown_script,0", + "tlvdata,accept_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", + "msgtype,funding_created,34", + "msgdata,funding_created,temporary_channel_id,byte,32", + "msgdata,funding_created,funding_txid,sha256,", + "msgdata,funding_created,funding_output_index,u16,", + "msgdata,funding_created,signature,signature,", + "msgtype,funding_signed,35", + "msgdata,funding_signed,channel_id,channel_id,", + "msgdata,funding_signed,signature,signature,", + "msgtype,funding_locked,36", + "msgdata,funding_locked,channel_id,channel_id,", + "msgdata,funding_locked,next_per_commitment_point,point,", + "msgtype,shutdown,38", + "msgdata,shutdown,channel_id,channel_id,", + "msgdata,shutdown,len,u16,", + "msgdata,shutdown,scriptpubkey,byte,len", + "msgtype,closing_signed,39", + "msgdata,closing_signed,channel_id,channel_id,", + "msgdata,closing_signed,fee_satoshis,u64,", + "msgdata,closing_signed,signature,signature,", + "msgtype,update_add_htlc,128", + "msgdata,update_add_htlc,channel_id,channel_id,", + "msgdata,update_add_htlc,id,u64,", + "msgdata,update_add_htlc,amount_msat,u64,", + "msgdata,update_add_htlc,payment_hash,sha256,", + "msgdata,update_add_htlc,cltv_expiry,u32,", + "msgdata,update_add_htlc,onion_routing_packet,byte,1366", + "msgtype,update_fulfill_htlc,130", + "msgdata,update_fulfill_htlc,channel_id,channel_id,", + "msgdata,update_fulfill_htlc,id,u64,", + "msgdata,update_fulfill_htlc,payment_preimage,byte,32", + "msgtype,update_fail_htlc,131", + "msgdata,update_fail_htlc,channel_id,channel_id,", + "msgdata,update_fail_htlc,id,u64,", + "msgdata,update_fail_htlc,len,u16,", + "msgdata,update_fail_htlc,reason,byte,len", + "msgtype,update_fail_malformed_htlc,135", + "msgdata,update_fail_malformed_htlc,channel_id,channel_id,", + "msgdata,update_fail_malformed_htlc,id,u64,", + "msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256,", + "msgdata,update_fail_malformed_htlc,failure_code,u16,", + "msgtype,commitment_signed,132", + "msgdata,commitment_signed,channel_id,channel_id,", + "msgdata,commitment_signed,signature,signature,", + "msgdata,commitment_signed,num_htlcs,u16,", + "msgdata,commitment_signed,htlc_signature,signature,num_htlcs", + "msgtype,revoke_and_ack,133", + "msgdata,revoke_and_ack,channel_id,channel_id,", + "msgdata,revoke_and_ack,per_commitment_secret,byte,32", + "msgdata,revoke_and_ack,next_per_commitment_point,point,", + "msgtype,update_fee,134", + "msgdata,update_fee,channel_id,channel_id,", + "msgdata,update_fee,feerate_per_kw,u32,", + "msgtype,channel_reestablish,136", + "msgdata,channel_reestablish,channel_id,channel_id,", + "msgdata,channel_reestablish,next_commitment_number,u64,", + "msgdata,channel_reestablish,next_revocation_number,u64,", + "msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32", + "msgdata,channel_reestablish,my_current_per_commitment_point,point,", +] +desc = "BOLT #2: Peer Protocol for Channel Management" +text = """# BOLT #2: Peer Protocol for Channel Management + +The peer channel protocol has three phases: establishment, normal +operation, and closing. + +# Table of Contents + + * [Channel](#channel) + * [Definition of `channel_id`](#definition-of-channel_id) + * [Channel Establishment](#channel-establishment) + * [The `open_channel` Message](#the-open_channel-message) + * [The `accept_channel` Message](#the-accept_channel-message) + * [The `funding_created` Message](#the-funding_created-message) + * [The `funding_signed` Message](#the-funding_signed-message) + * [The `funding_locked` Message](#the-funding_locked-message) + * [Channel Close](#channel-close) + * [Closing Initiation: `shutdown`](#closing-initiation-shutdown) + * [Closing Negotiation: `closing_signed`](#closing-negotiation-closing_signed) + * [Normal Operation](#normal-operation) + * [Forwarding HTLCs](#forwarding-htlcs) + * [`cltv_expiry_delta` Selection](#cltv_expiry_delta-selection) + * [Adding an HTLC: `update_add_htlc`](#adding-an-htlc-update_add_htlc) + * [Removing an HTLC: `update_fulfill_htlc`, `update_fail_htlc`, and `update_fail_malformed_htlc`](#removing-an-htlc-update_fulfill_htlc-update_fail_htlc-and-update_fail_malformed_htlc) + * [Committing Updates So Far: `commitment_signed`](#committing-updates-so-far-commitment_signed) + * [Completing the Transition to the Updated State: `revoke_and_ack`](#completing-the-transition-to-the-updated-state-revoke_and_ack) + * [Updating Fees: `update_fee`](#updating-fees-update_fee) + * [Message Retransmission: `channel_reestablish` message](#message-retransmission) + * [Authors](#authors) + +# Channel + +## Definition of `channel_id` + +Some messages use a `channel_id` to identify the channel. It's +derived from the funding transaction by combining the `funding_txid` +and the `funding_output_index`, using big-endian exclusive-OR +(i.e. `funding_output_index` alters the last 2 bytes). + +Prior to channel establishment, a `temporary_channel_id` is used, +which is a random nonce. + +Note that as duplicate `temporary_channel_id`s may exist from different +peers, APIs which reference channels by their channel id before the funding +transaction is created are inherently unsafe. The only protocol-provided +identifier for a channel before funding_created has been exchanged is the +(source_node_id, destination_node_id, temporary_channel_id) tuple. Note that +any such APIs which reference channels by their channel id before the funding +transaction is confirmed are also not persistent - until you know the script +pubkey corresponding to the funding output nothing prevents duplicative channel +ids. + + +## Channel Establishment + +After authenticating and initializing a connection ([BOLT #8](08-transport.md) +and [BOLT #1](01-messaging.md#the-init-message), respectively), channel establishment may begin. +This consists of the funding node (funder) sending an `open_channel` message, +followed by the responding node (fundee) sending `accept_channel`. With the +channel parameters locked in, the funder is able to create the funding +transaction and both versions of the commitment transaction, as described in +[BOLT #3](03-transactions.md#bolt-3-bitcoin-transaction-and-script-formats). +The funder then sends the outpoint of the funding output with the `funding_created` +message, along with the signature for the fundee's version of the commitment +transaction. Once the fundee learns the funding outpoint, it's able to +generate the signature for the funder's version of the commitment transaction and send it +over using the `funding_signed` message. + +Once the channel funder receives the `funding_signed` message, it +must broadcast the funding transaction to the Bitcoin network. After +the `funding_signed` message is sent/received, both sides should wait +for the funding transaction to enter the blockchain and reach the +specified depth (number of confirmations). After both sides have sent +the `funding_locked` message, the channel is established and can begin +normal operation. The `funding_locked` message includes information +that will be used to construct channel authentication proofs. + + + +-------+ +-------+ + | |--(1)--- open_channel ----->| | + | |<-(2)-- accept_channel -----| | + | | | | + | A |--(3)-- funding_created --->| B | + | |<-(4)-- funding_signed -----| | + | | | | + | |--(5)--- funding_locked ---->| | + | |<-(6)--- funding_locked -----| | + +-------+ +-------+ + + - where node A is 'funder' and node B is 'fundee' + +If this fails at any stage, or if one node decides the channel terms +offered by the other node are not suitable, the channel establishment +fails. + +Note that multiple channels can operate in parallel, as all channel +messages are identified by either a `temporary_channel_id` (before the +funding transaction is created) or a `channel_id` (derived from the +funding transaction). + +### The `open_channel` Message + +This message contains information about a node and indicates its +desire to set up a new channel. This is the first step toward creating +the funding transaction and both versions of the commitment transaction. + +1. type: 32 (`open_channel`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`32*byte`:`temporary_channel_id`] + * [`u64`:`funding_satoshis`] + * [`u64`:`push_msat`] + * [`u64`:`dust_limit_satoshis`] + * [`u64`:`max_htlc_value_in_flight_msat`] + * [`u64`:`channel_reserve_satoshis`] + * [`u64`:`htlc_minimum_msat`] + * [`u32`:`feerate_per_kw`] + * [`u16`:`to_self_delay`] + * [`u16`:`max_accepted_htlcs`] + * [`point`:`funding_pubkey`] + * [`point`:`revocation_basepoint`] + * [`point`:`payment_basepoint`] + * [`point`:`delayed_payment_basepoint`] + * [`point`:`htlc_basepoint`] + * [`point`:`first_per_commitment_point`] + * [`byte`:`channel_flags`] + * [`open_channel_tlvs`:`tlvs`] + +1. tlvs: `open_channel_tlvs` +2. types: + 1. type: 0 (`upfront_shutdown_script`) + 2. data: + * [`...*byte`:`shutdown_scriptpubkey`] + +The `chain_hash` value denotes the exact blockchain that the opened channel will +reside within. This is usually the genesis hash of the respective blockchain. +The existence of the `chain_hash` allows nodes to open channels +across many distinct blockchains as well as have channels within multiple +blockchains opened to the same peer (if it supports the target chains). + +The `temporary_channel_id` is used to identify this channel on a per-peer basis until the +funding transaction is established, at which point it is replaced +by the `channel_id`, which is derived from the funding transaction. + +`funding_satoshis` is the amount the sender is putting into the +channel. `push_msat` is an amount of initial funds that the sender is +unconditionally giving to the receiver. `dust_limit_satoshis` is the +threshold below which outputs should not be generated for this node's +commitment or HTLC transactions (i.e. HTLCs below this amount plus +HTLC transaction fees are not enforceable on-chain). This reflects the +reality that tiny outputs are not considered standard transactions and +will not propagate through the Bitcoin network. `channel_reserve_satoshis` +is the minimum amount that the other node is to keep as a direct +payment. `htlc_minimum_msat` indicates the smallest value HTLC this +node will accept. + +`max_htlc_value_in_flight_msat` is a cap on total value of outstanding +HTLCs, which allows a node to limit its exposure to HTLCs; similarly, +`max_accepted_htlcs` limits the number of outstanding HTLCs the other +node can offer. + +`feerate_per_kw` indicates the initial fee rate in satoshi per 1000-weight +(i.e. 1/4 the more normally-used 'satoshi per 1000 vbytes') that this +side will pay for commitment and HTLC transactions, as described in +[BOLT #3](03-transactions.md#fee-calculation) (this can be adjusted +later with an `update_fee` message). + +`to_self_delay` is the number of blocks that the other node's to-self +outputs must be delayed, using `OP_CHECKSEQUENCEVERIFY` delays; this +is how long it will have to wait in case of breakdown before redeeming +its own funds. + +`funding_pubkey` is the public key in the 2-of-2 multisig script of +the funding transaction output. + +The various `_basepoint` fields are used to derive unique +keys as described in [BOLT #3](03-transactions.md#key-derivation) for each commitment +transaction. Varying these keys ensures that the transaction ID of +each commitment transaction is unpredictable to an external observer, +even if one commitment transaction is seen; this property is very +useful for preserving privacy when outsourcing penalty transactions to +third parties. + +`first_per_commitment_point` is the per-commitment point to be used +for the first commitment transaction, + +Only the least-significant bit of `channel_flags` is currently +defined: `announce_channel`. This indicates whether the initiator of +the funding flow wishes to advertise this channel publicly to the +network, as detailed within [BOLT #7](07-routing-gossip.md#bolt-7-p2p-node-and-channel-discovery). + +The `shutdown_scriptpubkey` allows the sending node to commit to where +funds will go on mutual close, which the remote node should enforce +even if a node is compromised later. + +The `option_support_large_channel` is a feature used to let everyone +know this node will accept `funding_satoshis` greater than or equal to 2^24. +Since it's broadcast in the `node_announcement` message other nodes can use it to identify peers +willing to accept large channel even before exchanging the `init` message with them. + +#### Requirements + +The sending node: + - MUST ensure the `chain_hash` value identifies the chain it wishes to open the channel within. + - MUST ensure `temporary_channel_id` is unique from any other channel ID with the same peer. + - if both nodes advertised `option_support_large_channel`: + - MAY set `funding_satoshis` greater than or equal to 2^24 satoshi. + - otherwise: + - MUST set `funding_satoshis` to less than 2^24 satoshi. + - MUST set `push_msat` to equal or less than 1000 * `funding_satoshis`. + - MUST set `funding_pubkey`, `revocation_basepoint`, `htlc_basepoint`, `payment_basepoint`, and `delayed_payment_basepoint` to valid secp256k1 pubkeys in compressed format. + - MUST set `first_per_commitment_point` to the per-commitment point to be used for the initial commitment transaction, derived as specified in [BOLT #3](03-transactions.md#per-commitment-secret-requirements). + - MUST set `channel_reserve_satoshis` greater than or equal to `dust_limit_satoshis`. + - MUST set undefined bits in `channel_flags` to 0. + - if both nodes advertised the `option_upfront_shutdown_script` feature: + - MUST include `upfront_shutdown_script` with either a valid `shutdown_scriptpubkey` as required by `shutdown` `scriptpubkey`, or a zero-length `shutdown_scriptpubkey` (ie. `0x0000`). + - otherwise: + - MAY include `upfront_shutdown_script`. + - if it includes `open_channel_tlvs`: + - MUST include `upfront_shutdown_script`. + +The sending node SHOULD: + - set `to_self_delay` sufficient to ensure the sender can irreversibly spend a commitment transaction output, in case of misbehavior by the receiver. + - set `feerate_per_kw` to at least the rate it estimates would cause the transaction to be immediately included in a block. + - set `dust_limit_satoshis` to a sufficient value to allow commitment transactions to propagate through the Bitcoin network. + - set `htlc_minimum_msat` to the minimum value HTLC it's willing to accept from this peer. + +The receiving node MUST: + - ignore undefined bits in `channel_flags`. + - if the connection has been re-established after receiving a previous + `open_channel`, BUT before receiving a `funding_created` message: + - accept a new `open_channel` message. + - discard the previous `open_channel` message. + +The receiving node MAY fail the channel if: + - `announce_channel` is `false` (`0`), yet it wishes to publicly announce the channel. + - `funding_satoshis` is too small. + - it considers `htlc_minimum_msat` too large. + - it considers `max_htlc_value_in_flight_msat` too small. + - it considers `channel_reserve_satoshis` too large. + - it considers `max_accepted_htlcs` too small. + - it considers `dust_limit_satoshis` too small and plans to rely on the sending node publishing its commitment transaction in the event of a data loss (see [message-retransmission](02-peer-protocol.md#message-retransmission)). + +The receiving node MUST fail the channel if: + - the `chain_hash` value is set to a hash of a chain that is unknown to the receiver. + - `push_msat` is greater than `funding_satoshis` * 1000. + - `to_self_delay` is unreasonably large. + - `max_accepted_htlcs` is greater than 483. + - it considers `feerate_per_kw` too small for timely processing or unreasonably large. + - `funding_pubkey`, `revocation_basepoint`, `htlc_basepoint`, `payment_basepoint`, or `delayed_payment_basepoint` +are not valid secp256k1 pubkeys in compressed format. + - `dust_limit_satoshis` is greater than `channel_reserve_satoshis`. + - the funder's amount for the initial commitment transaction is not sufficient for full [fee payment](03-transactions.md#fee-payment). + - both `to_local` and `to_remote` amounts for the initial commitment transaction are less than or equal to `channel_reserve_satoshis` (see [BOLT 3](03-transactions.md#commitment-transaction-outputs)). + - `funding_satoshis` is greater than or equal to 2^24 and the receiver does not support `option_support_large_channel`. + +The receiving node MUST NOT: + - consider funds received, using `push_msat`, to be received until the funding transaction has reached sufficient depth. + +#### Rationale + +The requirement for `funding_satoshis` to be less than 2^24 satoshi was a temporary self-imposed limit while implementations were not yet considered stable, it can be lifted by advertising `option_support_large_channel`. + +The *channel reserve* is specified by the peer's `channel_reserve_satoshis`: 1% of the channel total is suggested. Each side of a channel maintains this reserve so it always has something to lose if it were to try to broadcast an old, revoked commitment transaction. Initially, this reserve may not be met, as only one side has funds; but the protocol ensures that there is always progress toward meeting this reserve, and once met, it is maintained. + +The sender can unconditionally give initial funds to the receiver using a non-zero `push_msat`, but even in this case we ensure that the funder has sufficient remaining funds to pay fees and that one side has some amount it can spend (which also implies there is at least one non-dust output). Note that, like any other on-chain transaction, this payment is not certain until the funding transaction has been confirmed sufficiently (with a danger of double-spend until this occurs) and may require a separate method to prove payment via on-chain confirmation. + +The `feerate_per_kw` is generally only of concern to the sender (who pays the fees), but there is also the fee rate paid by HTLC transactions; thus, unreasonably large fee rates can also penalize the recipient. + +Separating the `htlc_basepoint` from the `payment_basepoint` improves security: a node needs the secret associated with the `htlc_basepoint` to produce HTLC signatures for the protocol, but the secret for the `payment_basepoint` can be in cold storage. + +The requirement that `channel_reserve_satoshis` is not considered dust +according to `dust_limit_satoshis` eliminates cases where all outputs +would be eliminated as dust. The similar requirements in +`accept_channel` ensure that both sides' `channel_reserve_satoshis` +are above both `dust_limit_satoshis`. + +Details for how to handle a channel failure can be found in [BOLT 5:Failing a Channel](05-onchain.md#failing-a-channel). + +### The `accept_channel` Message + +This message contains information about a node and indicates its +acceptance of the new channel. This is the second step toward creating the +funding transaction and both versions of the commitment transaction. + +1. type: 33 (`accept_channel`) +2. data: + * [`32*byte`:`temporary_channel_id`] + * [`u64`:`dust_limit_satoshis`] + * [`u64`:`max_htlc_value_in_flight_msat`] + * [`u64`:`channel_reserve_satoshis`] + * [`u64`:`htlc_minimum_msat`] + * [`u32`:`minimum_depth`] + * [`u16`:`to_self_delay`] + * [`u16`:`max_accepted_htlcs`] + * [`point`:`funding_pubkey`] + * [`point`:`revocation_basepoint`] + * [`point`:`payment_basepoint`] + * [`point`:`delayed_payment_basepoint`] + * [`point`:`htlc_basepoint`] + * [`point`:`first_per_commitment_point`] + * [`accept_channel_tlvs`:`tlvs`] + +1. tlvs: `accept_channel_tlvs` +2. types: + 1. type: 0 (`upfront_shutdown_script`) + 2. data: + * [`...*byte`:`shutdown_scriptpubkey`] + +#### Requirements + +The `temporary_channel_id` MUST be the same as the `temporary_channel_id` in +the `open_channel` message. + +The sender: + - SHOULD set `minimum_depth` to a number of blocks it considers reasonable to +avoid double-spending of the funding transaction. + - MUST set `channel_reserve_satoshis` greater than or equal to `dust_limit_satoshis` from the `open_channel` message. + - MUST set `dust_limit_satoshis` less than or equal to `channel_reserve_satoshis` from the `open_channel` message. + +The receiver: + - if `minimum_depth` is unreasonably large: + - MAY reject the channel. + - if `channel_reserve_satoshis` is less than `dust_limit_satoshis` within the `open_channel` message: + - MUST reject the channel. + - if `channel_reserve_satoshis` from the `open_channel` message is less than `dust_limit_satoshis`: + - MUST reject the channel. +Other fields have the same requirements as their counterparts in `open_channel`. + +### The `funding_created` Message + +This message describes the outpoint which the funder has created for +the initial commitment transactions. After receiving the peer's +signature, via `funding_signed`, it will broadcast the funding transaction. + +1. type: 34 (`funding_created`) +2. data: + * [`32*byte`:`temporary_channel_id`] + * [`sha256`:`funding_txid`] + * [`u16`:`funding_output_index`] + * [`signature`:`signature`] + +#### Requirements + +The sender MUST set: + - `temporary_channel_id` the same as the `temporary_channel_id` in the `open_channel` message. + - `funding_txid` to the transaction ID of a non-malleable transaction, + - and MUST NOT broadcast this transaction. + - `funding_output_index` to the output number of that transaction that corresponds the funding transaction output, as defined in [BOLT #3](03-transactions.md#funding-transaction-output). + - `signature` to the valid signature using its `funding_pubkey` for the initial commitment transaction, as defined in [BOLT #3](03-transactions.md#commitment-transaction). + +The sender: + - when creating the funding transaction: + - SHOULD use only BIP141 (Segregated Witness) inputs. + +The recipient: + - if `signature` is incorrect: + - MUST fail the channel. + +#### Rationale + +The `funding_output_index` can only be 2 bytes, since that's how it's packed into the `channel_id` and used throughout the gossip protocol. The limit of 65535 outputs should not be overly burdensome. + +A transaction with all Segregated Witness inputs is not malleable, hence the funding transaction recommendation. + +### The `funding_signed` Message + +This message gives the funder the signature it needs for the first +commitment transaction, so it can broadcast the transaction knowing that funds +can be redeemed, if need be. + +This message introduces the `channel_id` to identify the channel. It's derived from the funding transaction by combining the `funding_txid` and the `funding_output_index`, using big-endian exclusive-OR (i.e. `funding_output_index` alters the last 2 bytes). + +1. type: 35 (`funding_signed`) +2. data: + * [`channel_id`:`channel_id`] + * [`signature`:`signature`] + +#### Requirements + +Both peers: + - if `option_static_remotekey` was negotiated: + - `option_static_remotekey` applies to all commitment transactions + - otherwise: + - `option_static_remotekey` does not apply to any commitment transactions + +The sender MUST set: + - `channel_id` by exclusive-OR of the `funding_txid` and the `funding_output_index` from the `funding_created` message. + - `signature` to the valid signature, using its `funding_pubkey` for the initial commitment transaction, as defined in [BOLT #3](03-transactions.md#commitment-transaction). + +The recipient: + - if `signature` is incorrect: + - MUST fail the channel. + - MUST NOT broadcast the funding transaction before receipt of a valid `funding_signed`. + - on receipt of a valid `funding_signed`: + - SHOULD broadcast the funding transaction. + +#### Rationale + +We decide on `option_static_remotekey` at this point when we first have to generate the commitment +transaction. Even if a later reconnection does not negotiate this parameter, this channel will continue to use `option_static_remotekey`; we don't support "downgrading". +This simplifies channel state, particularly penalty transaction handling. + +### The `funding_locked` Message + +This message indicates that the funding transaction has reached the `minimum_depth` asked for in `accept_channel`. Once both nodes have sent this, the channel enters normal operating mode. + +1. type: 36 (`funding_locked`) +2. data: + * [`channel_id`:`channel_id`] + * [`point`:`next_per_commitment_point`] + +#### Requirements + +The sender MUST: + - NOT send `funding_locked` unless outpoint of given by `funding_txid` and + `funding_output_index` in the `funding_created` message pays exactly `funding_satoshis` to the scriptpubkey specified in [BOLT #3](03-transactions.md#funding-transaction-output). + - wait until the funding transaction has reached +`minimum_depth` before sending this message. + - set `next_per_commitment_point` to the +per-commitment point to be used for the following commitment +transaction, derived as specified in +[BOLT #3](03-transactions.md#per-commitment-secret-requirements). + +A non-funding node (fundee): + - SHOULD forget the channel if it does not see the correct +funding transaction after a reasonable timeout. + +From the point of waiting for `funding_locked` onward, either node MAY +fail the channel if it does not receive a required response from the +other node after a reasonable timeout. + +#### Rationale + +The non-funder can simply forget the channel ever existed, since no +funds are at risk. If the fundee were to remember the channel forever, this +would create a Denial of Service risk; therefore, forgetting it is recommended +(even if the promise of `push_msat` is significant). + +## Channel Close + +Nodes can negotiate a mutual close of the connection, which unlike a +unilateral close, allows them to access their funds immediately and +can be negotiated with lower fees. + +Closing happens in two stages: +1. one side indicates it wants to clear the channel (and thus will accept no new HTLCs) +2. once all HTLCs are resolved, the final channel close negotiation begins. + + +-------+ +-------+ + | |--(1)----- shutdown ------->| | + | |<-(2)----- shutdown --------| | + | | | | + | | | | + | A | ... | B | + | | | | + | |--(3)-- closing_signed F1--->| | + | |<-(4)-- closing_signed F2----| | + | | ... | | + | |--(?)-- closing_signed Fn--->| | + | |<-(?)-- closing_signed Fn----| | + +-------+ +-------+ + +### Closing Initiation: `shutdown` + +Either node (or both) can send a `shutdown` message to initiate closing, +along with the `scriptpubkey` it wants to be paid to. + +1. type: 38 (`shutdown`) +2. data: + * [`channel_id`:`channel_id`] + * [`u16`:`len`] + * [`len*byte`:`scriptpubkey`] + +#### Requirements + +A sending node: + - if it hasn't sent a `funding_created` (if it is a funder) or a `funding_signed` (if it is a fundee): + - MUST NOT send a `shutdown` + - MAY send a `shutdown` before a `funding_locked`, i.e. before the funding transaction has reached `minimum_depth`. + - if there are updates pending on the receiving node's commitment transaction: + - MUST NOT send a `shutdown`. + - MUST NOT send an `update_add_htlc` after a `shutdown`. + - if no HTLCs remain in either commitment transaction: + - MUST NOT send any `update` message after a `shutdown`. + - SHOULD fail to route any HTLC added after it has sent `shutdown`. + - if it sent a non-zero-length `shutdown_scriptpubkey` in `open_channel` or `accept_channel`: + - MUST send the same value in `scriptpubkey`. + - MUST set `scriptpubkey` in one of the following forms: + + 1. `OP_DUP` `OP_HASH160` `20` 20-bytes `OP_EQUALVERIFY` `OP_CHECKSIG` + (pay to pubkey hash), OR + 2. `OP_HASH160` `20` 20-bytes `OP_EQUAL` (pay to script hash), OR + 3. `OP_0` `20` 20-bytes (version 0 pay to witness pubkey), OR + 4. `OP_0` `32` 32-bytes (version 0 pay to witness script hash) + +A receiving node: + - if it hasn't received a `funding_signed` (if it is a funder) or a `funding_created` (if it is a fundee): + - SHOULD fail the connection + - if the `scriptpubkey` is not in one of the above forms: + - SHOULD fail the connection. + - if it hasn't sent a `funding_locked` yet: + - MAY reply to a `shutdown` message with a `shutdown` + - once there are no outstanding updates on the peer, UNLESS it has already sent a `shutdown`: + - MUST reply to a `shutdown` message with a `shutdown` + - if both nodes advertised the `option_upfront_shutdown_script` feature, and the receiving node received a non-zero-length `shutdown_scriptpubkey` in `open_channel` or `accept_channel`, and that `shutdown_scriptpubkey` is not equal to `scriptpubkey`: + - MUST fail the connection. + +#### Rationale + +If channel state is always "clean" (no pending changes) when a +shutdown starts, the question of how to behave if it wasn't is avoided: +the sender always sends a `commitment_signed` first. + +As shutdown implies a desire to terminate, it implies that no new +HTLCs will be added or accepted. Once any HTLCs are cleared, the peer +may immediately begin closing negotiation, so we ban further updates +to the commitment transaction (in particular, `update_fee` would be +possible otherwise). + +The `scriptpubkey` forms include only standard forms accepted by the +Bitcoin network, which ensures the resulting transaction will +propagate to miners. + +The `option_upfront_shutdown_script` feature means that the node +wanted to pre-commit to `shutdown_scriptpubkey` in case it was +compromised somehow. This is a weak commitment (a malevolent +implementation tends to ignore specifications like this one!), but it +provides an incremental improvement in security by requiring the cooperation +of the receiving node to change the `scriptpubkey`. + +The `shutdown` response requirement implies that the node sends `commitment_signed` to commit any outstanding changes before replying; however, it could theoretically reconnect instead, which would simply erase all outstanding uncommitted changes. + +### Closing Negotiation: `closing_signed` + +Once shutdown is complete and the channel is empty of HTLCs, the final +current commitment transactions will have no HTLCs, and closing fee +negotiation begins. The funder chooses a fee it thinks is fair, and +signs the closing transaction with the `scriptpubkey` fields from the +`shutdown` messages (along with its chosen fee) and sends the signature; +the other node then replies similarly, using a fee it thinks is fair. This +exchange continues until both agree on the same fee or when one side fails +the channel. + +1. type: 39 (`closing_signed`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`fee_satoshis`] + * [`signature`:`signature`] + +#### Requirements + +The funding node: + - after `shutdown` has been received, AND no HTLCs remain in either commitment transaction: + - SHOULD send a `closing_signed` message. + +The sending node: + - MUST set `fee_satoshis` less than or equal to the + base fee of the final commitment transaction, as calculated in [BOLT #3](03-transactions.md#fee-calculation). + - SHOULD set the initial `fee_satoshis` according to its + estimate of cost of inclusion in a block. + - MUST set `signature` to the Bitcoin signature of the close + transaction, as specified in [BOLT #3](03-transactions.md#closing-transaction). + +The receiving node: + - if the `signature` is not valid for either variant of closing transaction + specified in [BOLT #3](03-transactions.md#closing-transaction): + - MUST fail the connection. + - if `fee_satoshis` is equal to its previously sent `fee_satoshis`: + - SHOULD sign and broadcast the final closing transaction. + - MAY close the connection. + - otherwise, if `fee_satoshis` is greater than +the base fee of the final commitment transaction as calculated in +[BOLT #3](03-transactions.md#fee-calculation): + - MUST fail the connection. + - if `fee_satoshis` is not strictly +between its last-sent `fee_satoshis` and its previously-received +`fee_satoshis`, UNLESS it has since reconnected: + - SHOULD fail the connection. + - if the receiver agrees with the fee: + - SHOULD reply with a `closing_signed` with the same `fee_satoshis` value. + - otherwise: + - MUST propose a value "strictly between" the received `fee_satoshis` + and its previously-sent `fee_satoshis`. + +#### Rationale + +The "strictly between" requirement ensures that forward +progress is made, even if only by a single satoshi at a time. To avoid +keeping state and to handle the corner case, where fees have shifted +between disconnection and reconnection, negotiation restarts on reconnection. + +Note there is limited risk if the closing transaction is +delayed, but it will be broadcast very soon; so there is usually no +reason to pay a premium for rapid processing. + +## Normal Operation + +Once both nodes have exchanged `funding_locked` (and optionally [`announcement_signatures`](07-routing-gossip.md#the-announcement_signatures-message)), the channel can be used to make payments via Hashed Time Locked Contracts. + +Changes are sent in batches: one or more `update_` messages are sent before a +`commitment_signed` message, as in the following diagram: + + +-------+ +-------+ + | |--(1)---- update_add_htlc ---->| | + | |--(2)---- update_add_htlc ---->| | + | |<-(3)---- update_add_htlc -----| | + | | | | + | |--(4)--- commitment_signed --->| | + | A |<-(5)---- revoke_and_ack ------| B | + | | | | + | |<-(6)--- commitment_signed ----| | + | |--(7)---- revoke_and_ack ----->| | + | | | | + | |--(8)--- commitment_signed --->| | + | |<-(9)---- revoke_and_ack ------| | + +-------+ +-------+ + +Counter-intuitively, these updates apply to the *other node's* +commitment transaction; the node only adds those updates to its own +commitment transaction when the remote node acknowledges it has +applied them via `revoke_and_ack`. + +Thus each update traverses through the following states: + +1. pending on the receiver +2. in the receiver's latest commitment transaction +3. ... and the receiver's previous commitment transaction has been revoked, + and the update is pending on the sender +4. ... and in the sender's latest commitment transaction +5. ... and the sender's previous commitment transaction has been revoked + + +As the two nodes' updates are independent, the two commitment +transactions may be out of sync indefinitely. This is not concerning: +what matters is whether both sides have irrevocably committed to a +particular update or not (the final state, above). + +### Forwarding HTLCs + +In general, a node offers HTLCs for two reasons: to initiate a payment of its own, +or to forward another node's payment. In the forwarding case, care must +be taken to ensure the *outgoing* HTLC cannot be redeemed unless the *incoming* +HTLC can be redeemed. The following requirements ensure this is always true. + +The respective **addition/removal** of an HTLC is considered *irrevocably committed* when: + +1. The commitment transaction **with/without** it is committed to by both nodes, and any +previous commitment transaction **without/with** it has been revoked, OR +2. The commitment transaction **with/without** it has been irreversibly committed to +the blockchain. + +#### Requirements + +A node: + - until an incoming HTLC has been irrevocably committed: + - MUST NOT offer the corresponding outgoing HTLC (`update_add_htlc`) in response to that incoming HTLC. + - until the removal of an outgoing HTLC is irrevocably committed, OR until the outgoing on-chain HTLC output has been spent via the HTLC-timeout transaction (with sufficient depth): + - MUST NOT fail the incoming HTLC (`update_fail_htlc`) that corresponds +to that outgoing HTLC. + - once the `cltv_expiry` of an incoming HTLC has been reached, OR if `cltv_expiry` minus `current_height` is less than `cltv_expiry_delta` for the corresponding outgoing HTLC: + - MUST fail that incoming HTLC (`update_fail_htlc`). + - if an incoming HTLC's `cltv_expiry` is unreasonably far in the future: + - SHOULD fail that incoming HTLC (`update_fail_htlc`). + - upon receiving an `update_fulfill_htlc` for an outgoing HTLC, OR upon discovering the `payment_preimage` from an on-chain HTLC spend: + - MUST fulfill the incoming HTLC that corresponds to that outgoing HTLC. + +#### Rationale + +In general, one side of the exchange needs to be dealt with before the other. +Fulfilling an HTLC is different: knowledge of the preimage is, by definition, +irrevocable and the incoming HTLC should be fulfilled as soon as possible to +reduce latency. + +An HTLC with an unreasonably long expiry is a denial-of-service vector and +therefore is not allowed. Note that the exact value of "unreasonable" is currently unclear +and may depend on network topology. + +### `cltv_expiry_delta` Selection + +Once an HTLC has timed out, it can either be fulfilled or timed-out; +care must be taken around this transition, both for offered and received HTLCs. + +Consider the following scenario, where A sends an HTLC to B, who +forwards to C, who delivers the goods as soon as the payment is +received. + +1. C needs to be sure that the HTLC from B cannot time out, even if B becomes + unresponsive; i.e. C can fulfill the incoming HTLC on-chain before B can + time it out on-chain. + +2. B needs to be sure that if C fulfills the HTLC from B, it can fulfill the + incoming HTLC from A; i.e. B can get the preimage from C and fulfill the incoming + HTLC on-chain before A can time it out on-chain. + +The critical settings here are the `cltv_expiry_delta` in +[BOLT #7](07-routing-gossip.md#the-channel_update-message) and the +related `min_final_cltv_expiry` in [BOLT #11](11-payment-encoding.md#tagged-fields). +`cltv_expiry_delta` is the minimum difference in HTLC CLTV timeouts, in +the forwarding case (B). `min_final_cltv_expiry` is the minimum difference +between HTLC CLTV timeout and the current block height, for the +terminal case (C). + +Note that a node is at risk if it accepts an HTLC in one channel and +offers an HTLC in another channel with too small of a difference between +the CLTV timeouts. For this reason, the `cltv_expiry_delta` for the +*outgoing* channel is used as the delta across a node. + +The worst-case number of blocks between outgoing and +incoming HTLC resolution can be derived, given a few assumptions: + +* a worst-case reorganization depth `R` blocks +* a grace-period `G` blocks after HTLC timeout before giving up on + an unresponsive peer and dropping to chain +* a number of blocks `S` between transaction broadcast and the + transaction being included in a block + +The worst case is for a forwarding node (B) that takes the longest +possible time to spot the outgoing HTLC fulfillment and also takes +the longest possible time to redeem it on-chain: + +1. The B->C HTLC times out at block `N`, and B waits `G` blocks until + it gives up waiting for C. B or C commits to the blockchain, + and B spends HTLC, which takes `S` blocks to be included. +2. Bad case: C wins the race (just) and fulfills the HTLC, B only sees + that transaction when it sees block `N+G+S+1`. +3. Worst case: There's reorganization `R` deep in which C wins and + fulfills. B only sees transaction at `N+G+S+R`. +4. B now needs to fulfill the incoming A->B HTLC, but A is unresponsive: B waits `G` more + blocks before giving up waiting for A. A or B commits to the blockchain. +5. Bad case: B sees A's commitment transaction in block `N+G+S+R+G+1` and has + to spend the HTLC output, which takes `S` blocks to be mined. +6. Worst case: there's another reorganization `R` deep which A uses to + spend the commitment transaction, so B sees A's commitment + transaction in block `N+G+S+R+G+R` and has to spend the HTLC output, which + takes `S` blocks to be mined. +7. B's HTLC spend needs to be at least `R` deep before it times out, + otherwise another reorganization could allow A to timeout the + transaction. + +Thus, the worst case is `3R+2G+2S`, assuming `R` is at least 1. Note that the +chances of three reorganizations in which the other node wins all of them is +low for `R` of 2 or more. Since high fees are used (and HTLC spends can use +almost arbitrary fees), `S` should be small; although, given that block times are +irregular and empty blocks still occur, `S=2` should be considered a +minimum. Similarly, the grace period `G` can be low (1 or 2), as nodes are +required to timeout or fulfill as soon as possible; but if `G` is too low it increases the +risk of unnecessary channel closure due to networking delays. + +There are four values that need be derived: + +1. the `cltv_expiry_delta` for channels, `3R+2G+2S`: if in doubt, a + `cltv_expiry_delta` of 12 is reasonable (R=2, G=1, S=2). + +2. the deadline for offered HTLCs: the deadline after which the channel has to be failed + and timed out on-chain. This is `G` blocks after the HTLC's + `cltv_expiry`: 1 block is reasonable. + +3. the deadline for received HTLCs this node has fulfilled: the deadline after which +the channel has to be failed and the HTLC fulfilled on-chain before its + `cltv_expiry`. See steps 4-7 above, which imply a deadline of `2R+G+S` + blocks before `cltv_expiry`: 7 blocks is reasonable. + +4. the minimum `cltv_expiry` accepted for terminal payments: the + worst case for the terminal node C is `2R+G+S` blocks (as, again, steps + 1-3 above don't apply). The default in + [BOLT #11](11-payment-encoding.md) is 9, which is slightly more + conservative than the 7 that this calculation suggests. + +#### Requirements + +An offering node: + - MUST estimate a timeout deadline for each HTLC it offers. + - MUST NOT offer an HTLC with a timeout deadline before its `cltv_expiry`. + - if an HTLC which it offered is in either node's current + commitment transaction, AND is past this timeout deadline: + - MUST fail the channel. + +A fulfilling node: + - for each HTLC it is attempting to fulfill: + - MUST estimate a fulfillment deadline. + - MUST fail (and not forward) an HTLC whose fulfillment deadline is already past. + - if an HTLC it has fulfilled is in either node's current commitment + transaction, AND is past this fulfillment deadline: + - MUST fail the channel. + +### Adding an HTLC: `update_add_htlc` + +Either node can send `update_add_htlc` to offer an HTLC to the other, +which is redeemable in return for a payment preimage. Amounts are in +millisatoshi, though on-chain enforcement is only possible for whole +satoshi amounts greater than the dust limit (in commitment transactions these are rounded down as +specified in [BOLT #3](03-transactions.md)). + +The format of the `onion_routing_packet` portion, which indicates where the payment +is destined, is described in [BOLT #4](04-onion-routing.md). + +1. type: 128 (`update_add_htlc`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`id`] + * [`u64`:`amount_msat`] + * [`sha256`:`payment_hash`] + * [`u32`:`cltv_expiry`] + * [`1366*byte`:`onion_routing_packet`] + +#### Requirements + +A sending node: + - if it is _responsible_ for paying the Bitcoin fee: + - MUST NOT offer `amount_msat` if, after adding that HTLC to its commitment + transaction, it cannot pay the fee for either the local or remote commitment + transaction at the current `feerate_per_kw` while maintaining its channel + reserve (see [Updating Fees](#updating-fees-update_fee)). + - SHOULD NOT offer `amount_msat` if, after adding that HTLC to its commitment + transaction, its remaining balance doesn't allow it to pay the commitment + transaction fee when receiving or sending a future additional non-dust HTLC + while maintaining its channel reserve. It is recommended that this "fee spike + buffer" can handle twice the current `feerate_per_kw` to ensure predictability + between implementations. + - if it is _not responsible_ for paying the Bitcoin fee: + - SHOULD NOT offer `amount_msat` if, once the remote node adds that HTLC to + its commitment transaction, it cannot pay the fee for the updated local or + remote transaction at the current `feerate_per_kw` while maintaining its + channel reserve. + - MUST offer `amount_msat` greater than 0. + - MUST NOT offer `amount_msat` below the receiving node's `htlc_minimum_msat` + - MUST set `cltv_expiry` less than 500000000. + - for channels with `chain_hash` identifying the Bitcoin blockchain: + - MUST set the four most significant bytes of `amount_msat` to 0. + - if result would be offering more than the remote's + `max_accepted_htlcs` HTLCs, in the remote commitment transaction: + - MUST NOT add an HTLC. + - if the sum of total offered HTLCs would exceed the remote's +`max_htlc_value_in_flight_msat`: + - MUST NOT add an HTLC. + - for the first HTLC it offers: + - MUST set `id` to 0. + - MUST increase the value of `id` by 1 for each successive offer. + +`id` MUST NOT be reset to 0 after the update is complete (i.e. after `revoke_and_ack` has +been received). It MUST continue incrementing instead. + +A receiving node: + - receiving an `amount_msat` equal to 0, OR less than its own `htlc_minimum_msat`: + - SHOULD fail the channel. + - receiving an `amount_msat` that the sending node cannot afford at the current `feerate_per_kw` (while maintaining its channel reserve): + - SHOULD fail the channel. + - if a sending node adds more than receiver `max_accepted_htlcs` HTLCs to + its local commitment transaction, OR adds more than receiver `max_htlc_value_in_flight_msat` worth of offered HTLCs to its local commitment transaction: + - SHOULD fail the channel. + - if sending node sets `cltv_expiry` to greater or equal to 500000000: + - SHOULD fail the channel. + - for channels with `chain_hash` identifying the Bitcoin blockchain, if the four most significant bytes of `amount_msat` are not 0: + - MUST fail the channel. + - MUST allow multiple HTLCs with the same `payment_hash`. + - if the sender did not previously acknowledge the commitment of that HTLC: + - MUST ignore a repeated `id` value after a reconnection. + - if other `id` violations occur: + - MAY fail the channel. + +The `onion_routing_packet` contains an obfuscated list of hops and instructions for each hop along the path. +It commits to the HTLC by setting the `payment_hash` as associated data, i.e. includes the `payment_hash` in the computation of HMACs. +This prevents replay attacks that would reuse a previous `onion_routing_packet` with a different `payment_hash`. + +#### Rationale + +Invalid amounts are a clear protocol violation and indicate a breakdown. + +If a node did not accept multiple HTLCs with the same payment hash, an +attacker could probe to see if a node had an existing HTLC. This +requirement, to deal with duplicates, leads to the use of a separate +identifier; it's assumed a 64-bit counter never wraps. + +Retransmissions of unacknowledged updates are explicitly allowed for +reconnection purposes; allowing them at other times simplifies the +recipient code (though strict checking may help debugging). + +`max_accepted_htlcs` is limited to 483 to ensure that, even if both +sides send the maximum number of HTLCs, the `commitment_signed` message will +still be under the maximum message size. It also ensures that +a single penalty transaction can spend the entire commitment transaction, +as calculated in [BOLT #5](05-onchain.md#penalty-transaction-weight-calculation). + +`cltv_expiry` values equal to or greater than 500000000 would indicate a time in +seconds, and the protocol only supports an expiry in blocks. + +`amount_msat` is deliberately limited for this version of the +specification; larger amounts are not necessary, nor wise, during the +bootstrap phase of the network. + +The node _responsible_ for paying the Bitcoin fee should maintain a "fee +spike buffer" on top of its reserve to accommodate a future fee increase. +Without this buffer, the node _responsible_ for paying the Bitcoin fee may +reach a state where it is unable to send or receive any non-dust HTLC while +maintaining its channel reserve (because of the increased weight of the +commitment transaction), resulting in a degraded channel. See [#728](https://github.com/lightningnetwork/lightning-rfc/issues/728) +for more details. + +### Removing an HTLC: `update_fulfill_htlc`, `update_fail_htlc`, and `update_fail_malformed_htlc` + +For simplicity, a node can only remove HTLCs added by the other node. +There are four reasons for removing an HTLC: the payment preimage is supplied, +it has timed out, it has failed to route, or it is malformed. + +To supply the preimage: + +1. type: 130 (`update_fulfill_htlc`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`id`] + * [`32*byte`:`payment_preimage`] + +For a timed out or route-failed HTLC: + +1. type: 131 (`update_fail_htlc`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`id`] + * [`u16`:`len`] + * [`len*byte`:`reason`] + +The `reason` field is an opaque encrypted blob for the benefit of the +original HTLC initiator, as defined in [BOLT #4](04-onion-routing.md); +however, there's a special malformed failure variant for the case where +the peer couldn't parse it: in this case the current node instead takes action, encrypting +it into a `update_fail_htlc` for relaying. + +For an unparsable HTLC: + +1. type: 135 (`update_fail_malformed_htlc`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`id`] + * [`sha256`:`sha256_of_onion`] + * [`u16`:`failure_code`] + +#### Requirements + +A node: + - SHOULD remove an HTLC as soon as it can. + - SHOULD fail an HTLC which has timed out. + - until the corresponding HTLC is irrevocably committed in both sides' + commitment transactions: + - MUST NOT send an `update_fulfill_htlc`, `update_fail_htlc`, or +`update_fail_malformed_htlc`. + +A receiving node: + - if the `id` does not correspond to an HTLC in its current commitment transaction: + - MUST fail the channel. + - if the `payment_preimage` value in `update_fulfill_htlc` + doesn't SHA256 hash to the corresponding HTLC `payment_hash`: + - MUST fail the channel. + - if the `BADONION` bit in `failure_code` is not set for + `update_fail_malformed_htlc`: + - MUST fail the channel. + - if the `sha256_of_onion` in `update_fail_malformed_htlc` doesn't match the + onion it sent: + - MAY retry or choose an alternate error response. + - otherwise, a receiving node which has an outgoing HTLC canceled by `update_fail_malformed_htlc`: + - MUST return an error in the `update_fail_htlc` sent to the link which + originally sent the HTLC, using the `failure_code` given and setting the + data to `sha256_of_onion`. + +#### Rationale + +A node that doesn't time out HTLCs risks channel failure (see +[`cltv_expiry_delta` Selection](#cltv_expiry_delta-selection)). + +A node that sends `update_fulfill_htlc`, before the sender, is also +committed to the HTLC and risks losing funds. + +If the onion is malformed, the upstream node won't be able to extract +the shared key to generate a response — hence the special failure message, which +makes this node do it. + +The node can check that the SHA256 that the upstream is complaining about +does match the onion it sent, which may allow it to detect random bit +errors. However, without re-checking the actual encrypted packet sent, +it won't know whether the error was its own or the remote's; so +such detection is left as an option. + +### Committing Updates So Far: `commitment_signed` + +When a node has changes for the remote commitment, it can apply them, +sign the resulting transaction (as defined in [BOLT #3](03-transactions.md)), and send a +`commitment_signed` message. + +1. type: 132 (`commitment_signed`) +2. data: + * [`channel_id`:`channel_id`] + * [`signature`:`signature`] + * [`u16`:`num_htlcs`] + * [`num_htlcs*signature`:`htlc_signature`] + +#### Requirements + +A sending node: + - MUST NOT send a `commitment_signed` message that does not include any +updates. + - MAY send a `commitment_signed` message that only +alters the fee. + - MAY send a `commitment_signed` message that doesn't +change the commitment transaction aside from the new revocation number +(due to dust, identical HTLC replacement, or insignificant or multiple +fee changes). + - MUST include one `htlc_signature` for every HTLC transaction corresponding + to the ordering of the commitment transaction (see [BOLT #3](03-transactions.md#transaction-input-and-output-ordering)). + - if it has not recently received a message from the remote node: + - SHOULD use `ping` and await the reply `pong` before sending `commitment_signed`. + +A receiving node: + - once all pending updates are applied: + - if `signature` is not valid for its local commitment transaction: + - MUST fail the channel. + - if `num_htlcs` is not equal to the number of HTLC outputs in the local + commitment transaction: + - MUST fail the channel. + - if any `htlc_signature` is not valid for the corresponding HTLC transaction: + - MUST fail the channel. + - MUST respond with a `revoke_and_ack` message. + +#### Rationale + +There's little point offering spam updates: it implies a bug. + +The `num_htlcs` field is redundant, but makes the packet length check fully self-contained. + +The recommendation to require recent messages recognizes the reality +that networks are unreliable: nodes might not realize their peers are +offline until after sending `commitment_signed`. Once +`commitment_signed` is sent, the sender considers itself bound to +those HTLCs, and cannot fail the related incoming HTLCs until the +output HTLCs are fully resolved. + +Note that the `htlc_signature` implicitly enforces the time-lock mechanism in the case of offered HTLCs being timed out or received HTLCs being spent. This is done to reduce fees by creating smaller scripts compared to explicitly stating time-locks on HTLC outputs. + +### Completing the Transition to the Updated State: `revoke_and_ack` + +Once the recipient of `commitment_signed` checks the signature and knows +it has a valid new commitment transaction, it replies with the commitment +preimage for the previous commitment transaction in a `revoke_and_ack` +message. + +This message also implicitly serves as an acknowledgment of receipt +of the `commitment_signed`, so this is a logical time for the `commitment_signed` sender +to apply (to its own commitment) any pending updates it sent before +that `commitment_signed`. + +The description of key derivation is in [BOLT #3](03-transactions.md#key-derivation). + +1. type: 133 (`revoke_and_ack`) +2. data: + * [`channel_id`:`channel_id`] + * [`32*byte`:`per_commitment_secret`] + * [`point`:`next_per_commitment_point`] + +#### Requirements + +A sending node: + - MUST set `per_commitment_secret` to the secret used to generate keys for + the previous commitment transaction. + - MUST set `next_per_commitment_point` to the values for its next commitment + transaction. + +A receiving node: + - if `per_commitment_secret` does not generate the previous `per_commitment_point`: + - MUST fail the channel. + - if the `per_commitment_secret` was not generated by the protocol in [BOLT #3](03-transactions.md#per-commitment-secret-requirements): + - MAY fail the channel. + +A node: + - MUST NOT broadcast old (revoked) commitment transactions, + - Note: doing so will allow the other node to seize all channel funds. + - SHOULD NOT sign commitment transactions, unless it's about to broadcast + them (due to a failed connection), + - Note: this is to reduce the above risk. + +### Updating Fees: `update_fee` + +An `update_fee` message is sent by the node which is paying the +Bitcoin fee. Like any update, it's first committed to the receiver's +commitment transaction and then (once acknowledged) committed to the +sender's. Unlike an HTLC, `update_fee` is never closed but simply +replaced. + +There is a possibility of a race, as the recipient can add new HTLCs +before it receives the `update_fee`. Under this circumstance, the sender may +not be able to afford the fee on its own commitment transaction, once the `update_fee` +is finally acknowledged by the recipient. In this case, the fee will be less +than the fee rate, as described in [BOLT #3](03-transactions.md#fee-payment). + +The exact calculation used for deriving the fee from the fee rate is +given in [BOLT #3](03-transactions.md#fee-calculation). + +1. type: 134 (`update_fee`) +2. data: + * [`channel_id`:`channel_id`] + * [`u32`:`feerate_per_kw`] + +#### Requirements + +The node _responsible_ for paying the Bitcoin fee: + - SHOULD send `update_fee` to ensure the current fee rate is sufficient (by a + significant margin) for timely processing of the commitment transaction. + +The node _not responsible_ for paying the Bitcoin fee: + - MUST NOT send `update_fee`. + +A receiving node: + - if the `update_fee` is too low for timely processing, OR is unreasonably large: + - SHOULD fail the channel. + - if the sender is not responsible for paying the Bitcoin fee: + - MUST fail the channel. + - if the sender cannot afford the new fee rate on the receiving node's + current commitment transaction: + - SHOULD fail the channel, + - but MAY delay this check until the `update_fee` is committed. + +#### Rationale + +Bitcoin fees are required for unilateral closes to be effective — +particularly since there is no general method for the broadcasting node to use +child-pays-for-parent to increase its effective fee. + +Given the variance in fees, and the fact that the transaction may be +spent in the future, it's a good idea for the fee payer to keep a good +margin (say 5x the expected fee requirement); but, due to differing methods of +fee estimation, an exact value is not specified. + +Since the fees are currently one-sided (the party which requested the +channel creation always pays the fees for the commitment transaction), +it's simplest to only allow it to set fee levels; however, as the same +fee rate applies to HTLC transactions, the receiving node must also +care about the reasonableness of the fee. + +## Message Retransmission + +Because communication transports are unreliable, and may need to be +re-established from time to time, the design of the transport has been +explicitly separated from the protocol. + +Nonetheless, it's assumed our transport is ordered and reliable. +Reconnection introduces doubt as to what has been received, so there are +explicit acknowledgments at that point. + +This is fairly straightforward in the case of channel establishment +and close, where messages have an explicit order, but during normal +operation, acknowledgments of updates are delayed until the +`commitment_signed` / `revoke_and_ack` exchange; so it cannot be assumed +that the updates have been received. This also means that the receiving +node only needs to store updates upon receipt of `commitment_signed`. + +Note that messages described in [BOLT #7](07-routing-gossip.md) are +independent of particular channels; their transmission requirements +are covered there, and besides being transmitted after `init` (as all +messages are), they are independent of requirements here. + +1. type: 136 (`channel_reestablish`) +2. data: + * [`channel_id`:`channel_id`] + * [`u64`:`next_commitment_number`] + * [`u64`:`next_revocation_number`] + * [`32*byte`:`your_last_per_commitment_secret`] + * [`point`:`my_current_per_commitment_point`] + +`next_commitment_number`: A commitment number is a 48-bit +incrementing counter for each commitment transaction; counters +are independent for each peer in the channel and start at 0. +They're only explicitly relayed to the other node in the case of +re-establishment, otherwise they are implicit. + +### Requirements + +A funding node: + - upon disconnection: + - if it has broadcast the funding transaction: + - MUST remember the channel for reconnection. + - otherwise: + - SHOULD NOT remember the channel for reconnection. + +A non-funding node: + - upon disconnection: + - if it has sent the `funding_signed` message: + - MUST remember the channel for reconnection. + - otherwise: + - SHOULD NOT remember the channel for reconnection. + +A node: + - MUST handle continuation of a previous channel on a new encrypted transport. + - upon disconnection: + - MUST reverse any uncommitted updates sent by the other side (i.e. all + messages beginning with `update_` for which no `commitment_signed` has + been received). + - Note: a node MAY have already used the `payment_preimage` value from + the `update_fulfill_htlc`, so the effects of `update_fulfill_htlc` are not + completely reversed. + - upon reconnection: + - if a channel is in an error state: + - SHOULD retransmit the error packet and ignore any other packets for + that channel. + - otherwise: + - MUST transmit `channel_reestablish` for each channel. + - MUST wait to receive the other node's `channel_reestablish` + message before sending any other messages for that channel. + +The sending node: + - MUST set `next_commitment_number` to the commitment number of the + next `commitment_signed` it expects to receive. + - MUST set `next_revocation_number` to the commitment number of the + next `revoke_and_ack` message it expects to receive. + - if `option_static_remotekey` applies to the commitment transaction: + - MUST set `my_current_per_commitment_point` to a valid point. + - otherwise: + - MUST set `my_current_per_commitment_point` to its commitment point for + the last signed commitment it received from its channel peer (i.e. the commitment_point + corresponding to the commitment transaction the sender would use to unilaterally close). + - if `next_revocation_number` equals 0: + - MUST set `your_last_per_commitment_secret` to all zeroes + - otherwise: + - MUST set `your_last_per_commitment_secret` to the last `per_commitment_secret` it received + +A node: + - if `next_commitment_number` is 1 in both the `channel_reestablish` it + sent and received: + - MUST retransmit `funding_locked`. + - otherwise: + - MUST NOT retransmit `funding_locked`. + - upon reconnection: + - MUST ignore any redundant `funding_locked` it receives. + - if `next_commitment_number` is equal to the commitment number of + the last `commitment_signed` message the receiving node has sent: + - MUST reuse the same commitment number for its next `commitment_signed`. + - otherwise: + - if `next_commitment_number` is not 1 greater than the + commitment number of the last `commitment_signed` message the receiving + node has sent: + - SHOULD fail the channel. + - if it has not sent `commitment_signed`, AND `next_commitment_number` + is not equal to 1: + - SHOULD fail the channel. + - if `next_revocation_number` is equal to the commitment number of + the last `revoke_and_ack` the receiving node sent, AND the receiving node + hasn't already received a `closing_signed`: + - MUST re-send the `revoke_and_ack`. + - otherwise: + - if `next_revocation_number` is not equal to 1 greater than the + commitment number of the last `revoke_and_ack` the receiving node has sent: + - SHOULD fail the channel. + - if it has not sent `revoke_and_ack`, AND `next_revocation_number` + is not equal to 0: + - SHOULD fail the channel. + + A receiving node: + - if `option_static_remotekey` applies to the commitment transaction: + - if `next_revocation_number` is greater than expected above, AND + `your_last_per_commitment_secret` is correct for that + `next_revocation_number` minus 1: + - MUST NOT broadcast its commitment transaction. + - SHOULD fail the channel. + - otherwise: + - if `your_last_per_commitment_secret` does not match the expected values: + - SHOULD fail the channel. + - otherwise, if it supports `option_data_loss_protect`: + - if `next_revocation_number` is greater than expected above, AND + `your_last_per_commitment_secret` is correct for that + `next_revocation_number` minus 1: + - MUST NOT broadcast its commitment transaction. + - SHOULD fail the channel. + - SHOULD store `my_current_per_commitment_point` to retrieve funds + should the sending node broadcast its commitment transaction on-chain. + - otherwise (`your_last_per_commitment_secret` or `my_current_per_commitment_point` + do not match the expected values): + - SHOULD fail the channel. + +A node: + - MUST NOT assume that previously-transmitted messages were lost, + - if it has sent a previous `commitment_signed` message: + - MUST handle the case where the corresponding commitment transaction is + broadcast at any time by the other side, + - Note: this is particularly important if the node does not simply + retransmit the exact `update_` messages as previously sent. + - upon reconnection: + - if it has sent a previous `shutdown`: + - MUST retransmit `shutdown`. + +### Rationale + +The requirements above ensure that the opening phase is nearly +atomic: if it doesn't complete, it starts again. The only exception +is if the `funding_signed` message is sent but not received. In +this case, the funder will forget the channel, and presumably open +a new one upon reconnection; meanwhile, the other node will eventually forget +the original channel, due to never receiving `funding_locked` or seeing +the funding transaction on-chain. + +There's no acknowledgment for `error`, so if a reconnect occurs it's +polite to retransmit before disconnecting again; however, it's not a MUST, +because there are also occasions where a node can simply forget the +channel altogether. + +`closing_signed` also has no acknowledgment so must be retransmitted +upon reconnection (though negotiation restarts on reconnection, so it needs +not be an exact retransmission). +The only acknowledgment for `shutdown` is `closing_signed`, so one or the other +needs to be retransmitted. + +The handling of updates is similarly atomic: if the commit is not +acknowledged (or wasn't sent) the updates are re-sent. However, it's not +insisted they be identical: they could be in a different order, +involve different fees, or even be missing HTLCs which are now too old +to be added. Requiring they be identical would effectively mean a +write to disk by the sender upon each transmission, whereas the scheme +here encourages a single persistent write to disk for each +`commitment_signed` sent or received. + +A re-transmittal of `revoke_and_ack` should never be asked for after a +`closing_signed` has been received, since that would imply a shutdown has been +completed — which can only occur after the `revoke_and_ack` has been received +by the remote node. + +Note that the `next_commitment_number` starts at 1, since +commitment number 0 is created during opening. +`next_revocation_number` will be 0 until the +`commitment_signed` for commitment number 1 is send and then +the revocation for commitment number 0 is received. + +`funding_locked` is implicitly acknowledged by the start of normal +operation, which is known to have begun after a `commitment_signed` has been +received — hence, the test for a `next_commitment_number` greater +than 1. + +A previous draft insisted that the funder "MUST remember ...if it has +broadcast the funding transaction, otherwise it MUST NOT": this was in +fact an impossible requirement. A node must either firstly commit to +disk and secondly broadcast the transaction or vice versa. The new +language reflects this reality: it's surely better to remember a +channel which hasn't been broadcast than to forget one which has! +Similarly, for the fundee's `funding_signed` message: it's better to +remember a channel that never opens (and times out) than to let the +funder open it while the fundee has forgotten it. + +`option_data_loss_protect` was added to allow a node, which has somehow fallen behind +(e.g. has been restored from old backup), to detect that it's fallen-behind. A fallen-behind +node must know it cannot broadcast its current commitment transaction — which would lead to +total loss of funds — as the remote node can prove it knows the +revocation preimage. The error returned by the fallen-behind node +(or simply the invalid numbers in the `channel_reestablish` it has +sent) should make the other node drop its current commitment +transaction to the chain. This will, at least, allow the fallen-behind node to recover +non-HTLC funds, if the `my_current_per_commitment_point` +is valid. However, this also means the fallen-behind node has revealed this +fact (though not provably: it could be lying), and the other node could use this to +broadcast a previous state. + +`option_static_remotekey` removes the changing `to_remote` key, +so the `my_current_per_commitment_point` is unnecessary and thus +ignored (for parsing simplicity, it remains and must be a valid point, +however), but the disclosure of previous secret still allows +fall-behind detection. An implementation can offer both, however, and +fall back to the `option_data_loss_protect` behavior if +`option_static_remotekey` is not negotiated. + +# Authors + +[ FIXME: Insert Author List ] + +![Creative Commons License](https://i.creativecommons.org/l/by/4.0/88x31.png "License CC-BY") +
+This work is licensed under a [Creative Commons Attribution 4.0 International License](http://creativecommons.org/licenses/by/4.0/). +""" diff --git a/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py new file mode 100644 index 000000000..92610b074 --- /dev/null +++ b/contrib/pyln-spec/bolt2/pyln/spec/bolt2/gen_version.py @@ -0,0 +1,2 @@ +__version__ = "1.0.post137" +__gitversion__ = "ae2d248b7ad8b0965f224c303019ba04c661008f" diff --git a/contrib/pyln-spec/bolt2/requirements.txt b/contrib/pyln-spec/bolt2/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/contrib/pyln-spec/bolt2/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/contrib/pyln-spec/bolt2/setup.py b/contrib/pyln-spec/bolt2/setup.py new file mode 100644 index 000000000..17c82edf5 --- /dev/null +++ b/contrib/pyln-spec/bolt2/setup.py @@ -0,0 +1,23 @@ +from pyln.spec.bolt2 import __version__, desc +from setuptools import setup +import io + +with io.open('requirements.txt', encoding='utf-8') as f: + requirements = [r for r in f.read().split('\n') if len(r)] + + +def do_setup(boltnum: int, version: str, desc: str): + setup(name='pyln-bolt{}'.format(boltnum), + version=version, + description=desc, + url='http://github.com/ElementsProject/lightning', + author='Rusty Russell', + author_email='rusty@rustcorp.com.au', + license='MIT', + packages=['pyln.spec.bolt{}'.format(boltnum)], + scripts=[], + zip_safe=True, + install_requires=requirements) + + +do_setup(2, __version__, desc) diff --git a/contrib/pyln-proto/tests/test_bolt2.py b/contrib/pyln-spec/bolt2/tests/test_bolt2.py similarity index 78% rename from contrib/pyln-proto/tests/test_bolt2.py rename to contrib/pyln-spec/bolt2/tests/test_bolt2.py index 7068b5803..50f6d1303 100644 --- a/contrib/pyln-proto/tests/test_bolt2.py +++ b/contrib/pyln-spec/bolt2/tests/test_bolt2.py @@ -1,6 +1,6 @@ #! /usr/bin/python3 from pyln.proto.message import MessageNamespace -import pyln.proto.message.bolt2 as bolt2 +import pyln.spec.bolt2 as bolt2 # FIXME: more tests diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/__init__.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/__init__.py new file mode 120000 index 000000000..52f300f8a --- /dev/null +++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/__init__.py @@ -0,0 +1 @@ +../../../../subinit.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/bolt.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/bolt.py new file mode 120000 index 000000000..c22b879fc --- /dev/null +++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/bolt.py @@ -0,0 +1 @@ +../../../../bolt.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen.py new file mode 100644 index 000000000..4c27b81c5 --- /dev/null +++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen.py @@ -0,0 +1,1267 @@ +csv = [ + "tlvtype,tlv_payload,amt_to_forward,2", + "tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64,", + "tlvtype,tlv_payload,outgoing_cltv_value,4", + "tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32,", + "tlvtype,tlv_payload,short_channel_id,6", + "tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id,", + "tlvtype,tlv_payload,payment_data,8", + "tlvdata,tlv_payload,payment_data,payment_secret,byte,32", + "tlvdata,tlv_payload,payment_data,total_msat,tu64,", + "msgtype,invalid_realm,PERM|1", + "msgtype,temporary_node_failure,NODE|2", + "msgtype,permanent_node_failure,PERM|NODE|2", + "msgtype,required_node_feature_missing,PERM|NODE|3", + "msgtype,invalid_onion_version,BADONION|PERM|4", + "msgdata,invalid_onion_version,sha256_of_onion,sha256,", + "msgtype,invalid_onion_hmac,BADONION|PERM|5", + "msgdata,invalid_onion_hmac,sha256_of_onion,sha256,", + "msgtype,invalid_onion_key,BADONION|PERM|6", + "msgdata,invalid_onion_key,sha256_of_onion,sha256,", + "msgtype,temporary_channel_failure,UPDATE|7", + "msgdata,temporary_channel_failure,len,u16,", + "msgdata,temporary_channel_failure,channel_update,byte,len", + "msgtype,permanent_channel_failure,PERM|8", + "msgtype,required_channel_feature_missing,PERM|9", + "msgtype,unknown_next_peer,PERM|10", + "msgtype,amount_below_minimum,UPDATE|11", + "msgdata,amount_below_minimum,htlc_msat,u64,", + "msgdata,amount_below_minimum,len,u16,", + "msgdata,amount_below_minimum,channel_update,byte,len", + "msgtype,fee_insufficient,UPDATE|12", + "msgdata,fee_insufficient,htlc_msat,u64,", + "msgdata,fee_insufficient,len,u16,", + "msgdata,fee_insufficient,channel_update,byte,len", + "msgtype,incorrect_cltv_expiry,UPDATE|13", + "msgdata,incorrect_cltv_expiry,cltv_expiry,u32,", + "msgdata,incorrect_cltv_expiry,len,u16,", + "msgdata,incorrect_cltv_expiry,channel_update,byte,len", + "msgtype,expiry_too_soon,UPDATE|14", + "msgdata,expiry_too_soon,len,u16,", + "msgdata,expiry_too_soon,channel_update,byte,len", + "msgtype,incorrect_or_unknown_payment_details,PERM|15", + "msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64,", + "msgdata,incorrect_or_unknown_payment_details,height,u32,", + "msgtype,final_incorrect_cltv_expiry,18", + "msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32,", + "msgtype,final_incorrect_htlc_amount,19", + "msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,", + "msgtype,channel_disabled,UPDATE|20", + "msgtype,expiry_too_far,21", + "msgtype,invalid_onion_payload,PERM|22", + "msgdata,invalid_onion_payload,type,bigsize,", + "msgdata,invalid_onion_payload,offset,u16,", + "msgtype,mpp_timeout,23", +] +desc = "BOLT #4: Onion Routing Protocol" +text = """# BOLT #4: Onion Routing Protocol + +## Overview + +This document describes the construction of an onion routed packet that is +used to route a payment from an _origin node_ to a _final node_. The packet +is routed through a number of intermediate nodes, called _hops_. + +The routing schema is based on the [Sphinx][sphinx] construction and is +extended with a per-hop payload. + +Intermediate nodes forwarding the message can verify the integrity of +the packet and can learn which node they should forward the +packet to. They cannot learn which other nodes, besides their +predecessor or successor, are part of the packet's route; nor can they learn +the length of the route or their position within it. The packet is +obfuscated at each hop, to ensure that a network-level attacker cannot +associate packets belonging to the same route (i.e. packets belonging +to the same route do not share any correlating information). Notice that this +does not preclude the possibility of packet association by an attacker +via traffic analysis. + +The route is constructed by the origin node, which knows the public +keys of each intermediate node and of the final node. Knowing each node's public key +allows the origin node to create a shared secret (using ECDH) for each +intermediate node and for the final node. The shared secret is then +used to generate a _pseudo-random stream_ of bytes (which is used to obfuscate +the packet) and a number of _keys_ (which are used to encrypt the payload and +compute the HMACs). The HMACs are then in turn used to ensure the integrity of +the packet at each hop. + +Each hop along the route only sees an ephemeral key for the origin node, in +order to hide the sender's identity. The ephemeral key is blinded by each +intermediate hop before forwarding to the next, making the onions unlinkable +along the route. + +This specification describes _version 0_ of the packet format and routing +mechanism. + +A node: + - upon receiving a higher version packet than it implements: + - MUST report a route failure to the origin node. + - MUST discard the packet. + +# Table of Contents + + * [Conventions](#conventions) + * [Key Generation](#key-generation) + * [Pseudo Random Byte Stream](#pseudo-random-byte-stream) + * [Packet Structure](#packet-structure) + * [Legacy HopData Payload Format](#legacy-hop_data-payload-format) + * [TLV Payload Format](#tlv_payload-format) + * [Basic Multi-Part Payments](#basic-multi-part-payments) + * [Accepting and Forwarding a Payment](#accepting-and-forwarding-a-payment) + * [Payload for the Last Node](#payload-for-the-last-node) + * [Non-strict Forwarding](#non-strict-forwarding) + * [Shared Secret](#shared-secret) + * [Blinding Ephemeral Keys](#blinding-ephemeral-keys) + * [Packet Construction](#packet-construction) + * [Packet Forwarding](#packet-forwarding) + * [Filler Generation](#filler-generation) + * [Returning Errors](#returning-errors) + * [Failure Messages](#failure-messages) + * [Receiving Failure Codes](#receiving-failure-codes) + * [Test Vector](#test-vector) + * [Returning Errors](#returning-errors) + * [References](#references) + * [Authors](#authors) + +# Conventions + +There are a number of conventions adhered to throughout this document: + + - HMAC: the integrity verification of the packet is based on Keyed-Hash + Message Authentication Code, as defined by the [FIPS 198 + Standard][fips198]/[RFC 2104][RFC2104], and using a `SHA256` hashing + algorithm. + - Elliptic curve: for all computations involving elliptic curves, the Bitcoin + curve is used, as specified in [`secp256k1`][sec2] + - Pseudo-random stream: [`ChaCha20`][rfc7539] is used to generate a + pseudo-random byte stream. For its generation, a fixed null-nonce + (`0x0000000000000000`) is used, along with a key derived from a shared + secret and with a `0x00`-byte stream of the desired output size as the + message. + - The terms _origin node_ and _final node_ refer to the initial packet sender + and the final packet recipient, respectively. + - The terms _hop_ and _node_ are sometimes used interchangeably, but a _hop_ + usually refers to an intermediate node in the route rather than an end node. + _origin node_ --> _hop_ --> ... --> _hop_ --> _final node_ + - The term _processing node_ refers to the specific node along the route that is + currently processing the forwarded packet. + - The term _peers_ refers only to hops that are direct neighbors (in the + overlay network): more specifically, _sending peers_ forward packets + to _receiving peers_. + - Each hop in the route has a variable length `hop_payload`, or a fixed-size + legacy `hop_data` payload. + - The legacy `hop_data` is identified by a single `0x00`-byte prefix + - The variable length `hop_payload` is prefixed with a `bigsize` encoding + the length in bytes, excluding the prefix and the trailing HMAC. + +# Key Generation + +A number of encryption and verification keys are derived from the shared secret: + + - _rho_: used as key when generating the pseudo-random byte stream that is used + to obfuscate the per-hop information + - _mu_: used during the HMAC generation + - _um_: used during error reporting + - _pad_: use to generate random filler bytes for the starting mix-header + packet + +The key generation function takes a key-type (_rho_=`0x72686F`, _mu_=`0x6d75`, +_um_=`0x756d`, or _pad_=`0x706164`) and a 32-byte secret as inputs and returns +a 32-byte key. + +Keys are generated by computing an HMAC (with `SHA256` as hashing algorithm) +using the appropriate key-type (i.e. _rho_, _mu_, _um_, or _pad_) as HMAC-key +and the 32-byte shared secret as the message. The resulting HMAC is then +returned as the key. + +Notice that the key-type does not include a C-style `0x00`-termination-byte, +e.g. the length of the _rho_ key-type is 3 bytes, not 4. + +# Pseudo Random Byte Stream + +The pseudo-random byte stream is used to obfuscate the packet at each hop of the +path, so that each hop may only recover the address and HMAC of the next hop. +The pseudo-random byte stream is generated by encrypting (using `ChaCha20`) a +`0x00`-byte stream, of the required length, which is initialized with a key +derived from the shared secret and a zero-nonce (`0x00000000000000`). + +The use of a fixed nonce is safe, since the keys are never reused. + +# Packet Structure + +The packet consists of four sections: + + - a `version` byte + - a 33-byte compressed `secp256k1` `public_key`, used during the shared secret + generation + - a 1300-byte `hop_payloads` consisting of multiple, variable length, + `hop_payload` payloads or up to 20 fixed sized legacy `hop_data` payloads. + - a 32-byte `hmac`, used to verify the packet's integrity + +The network format of the packet consists of the individual sections +serialized into one contiguous byte-stream and then transferred to the packet +recipient. Due to the fixed size of the packet, it need not be prefixed by its +length when transferred over a connection. + +The overall structure of the packet is as follows: + +1. type: `onion_packet` +2. data: + * [`byte`:`version`] + * [`point`:`public_key`] + * [`1300*byte`:`hop_payloads`] + * [`32*byte`:`hmac`] + +For this specification (_version 0_), `version` has a constant value of `0x00`. + +The `hop_payloads` field is a structure that holds obfuscated routing information, and associated HMAC. +It is 1300 bytes long and has the following structure: + +1. type: `hop_payloads` +2. data: + * [`bigsize`:`length`] + * [`hop_payload_length`:`hop_payload`] + * [`32*byte`:`hmac`] + * ... + * `filler` + +Where, the `length`, `hop_payload` (with contents dependent on `length`), and `hmac` are repeated for each hop; +and where, `filler` consists of obfuscated, deterministically-generated padding, as detailed in [Filler Generation](#filler-generation). +Additionally, `hop_payloads` is incrementally obfuscated at each hop. + +Using the `hop_payload` field, the origin node is able to specify the path and structure of the HTLCs forwarded at each hop. +As the `hop_payload` is protected under the packet-wide HMAC, the information it contains is fully authenticated with each pair-wise relationship between the HTLC sender (origin node) and each hop in the path. + +Using this end-to-end authentication, each hop is able to cross-check the HTLC +parameters with the `hop_payload`'s specified values and to ensure that the +sending peer hasn't forwarded an ill-crafted HTLC. + +The `length` field determines both the length and the format of the `hop_payload` field; the following formats are defined: + + - Legacy `hop_data` format, identified by a single `0x00` byte for length. In this case the `hop_payload_length` is defined to be 32 bytes. + - `tlv_payload` format, identified by any length over `1`. In this case the `hop_payload_length` is equal to the numeric value of `length`. + - A single `0x01` byte for length is reserved for future use to signal a different payload format. This is safe since no TLV value can ever be shorter than 2 bytes. In this case the `hop_payload_length` MUST be defined in the future specification making use of this `length`. + +## Legacy `hop_data` payload format + +The `hop_data` format is identified by a single `0x00`-byte length, for backward compatibility. +Its payload is defined as: + +1. type: `hop_data` (for `realm` 0) +2. data: + * [`short_channel_id`:`short_channel_id`] + * [`u64`:`amt_to_forward`] + * [`u32`:`outgoing_cltv_value`] + * [`12*byte`:`padding`] + +Field descriptions: + + * `short_channel_id`: The ID of the outgoing channel used to route the + message; the receiving peer should operate the other end of this channel. + + * `amt_to_forward`: The amount, in millisatoshis, to forward to the next + receiving peer specified within the routing information. + + For non-final nodes, this value amount MUST include the origin node's computed _fee_ for the + receiving peer. When processing an incoming Sphinx packet and the HTLC + message that it is encapsulated within, if the following inequality doesn't hold, + then the HTLC should be rejected as it would indicate that a prior hop has + deviated from the specified parameters: + + incoming_htlc_amt - fee >= amt_to_forward + + Where `fee` is calculated according to the receiving peer's advertised fee + schema (as described in [BOLT #7](07-routing-gossip.md#htlc-fees)). + + For the final node, this value MUST be exactly equal to the incoming htlc + amount, otherwise the HTLC should be rejected. + + * `outgoing_cltv_value`: The CLTV value that the _outgoing_ HTLC carrying + the packet should have. + + cltv_expiry - cltv_expiry_delta >= outgoing_cltv_value + + Inclusion of this field allows a hop to both authenticate the information + specified by the origin node, and the parameters of the HTLC forwarded, + and ensure the origin node is using the current `cltv_expiry_delta` value. + If there is no next hop, `cltv_expiry_delta` is 0. + If the values don't correspond, then the HTLC should be failed and rejected, as + this indicates that either a forwarding node has tampered with the intended HTLC + values or that the origin node has an obsolete `cltv_expiry_delta` value. + The hop MUST be consistent in responding to an unexpected + `outgoing_cltv_value`, whether it is the final node or not, to avoid + leaking its position in the route. + + * `padding`: This field is for future use and also for ensuring that future non-0-`realm` + `hop_data`s won't change the overall `hop_payloads` size. + +When forwarding HTLCs, nodes MUST construct the outgoing HTLC as specified +within `hop_data` above; otherwise, deviation from the specified HTLC +parameters may lead to extraneous routing failure. + +### `tlv_payload` format + +This is a more flexible format, which avoids the redundant `short_channel_id` field for the final node. + +1. tlvs: `tlv_payload` +2. types: + 1. type: 2 (`amt_to_forward`) + 2. data: + * [`tu64`:`amt_to_forward`] + 1. type: 4 (`outgoing_cltv_value`) + 2. data: + * [`tu32`:`outgoing_cltv_value`] + 1. type: 6 (`short_channel_id`) + 2. data: + * [`short_channel_id`:`short_channel_id`] + 1. type: 8 (`payment_data`) + 2. data: + * [`32*byte`:`payment_secret`] + * [`tu64`:`total_msat`] + +### Requirements + +The writer: + - Unless `node_announcement`, `init` message or the [BOLT #11](11-payment-encoding.md#tagged-fields) offers feature `var_onion_optin`: + - MUST use the legacy payload format instead. + - For every node: + - MUST include `amt_to_forward` and `outgoing_cltv_value`. + - For every non-final node: + - MUST include `short_channel_id` + - MUST NOT include `payment_data` + - For the final node: + - MUST NOT include `short_channel_id` + - if the recipient provided `payment_secret`: + - MUST include `payment_data` + - MUST set `payment_secret` to the one provided + - MUST set `total_msat` to the total amount it will send + +The reader: + - MUST return an error if `amt_to_forward` or `outgoing_cltv_value` are not present. + - if it is the final node: + - MUST treat `total_msat` as if it were equal to `amt_to_forward` if it + is not present. + +The requirements for the contents of these fields are specified [above](#legacy-hop_data-payload-format) +and [below](#basic-multi-part-payments). + +### Basic Multi-Part Payments + +An HTLC may be part of a larger "multi-part" payment: such +"base" atomic multipath payments will use the same `payment_hash` for +all paths. + +Note that `amt_to_forward` is the amount for this HTLC only: a +`total_msat` field containing a greater value is a promise by the +ultimate sender that the rest of the payment will follow in succeeding +HTLCs; we call these outstanding HTLCs which have the same preimage, +an "HTLC set". + +#### Requirements + +The writer: + - if the invoice offers the `basic_mpp` feature: + - MAY send more than one HTLC to pay the invoice. + - MUST use the same `payment_hash` on all HTLCs in the set. + - SHOULD send all payments at approximately the same time. + - SHOULD try to use diverse paths to the recipient for each HTLC. + - SHOULD retry and/or re-divide HTLCs which fail. + - if the invoice specifies an `amount`: + - MUST set `total_msat` to at least that `amount`, and less + than or equal to twice `amount`. + - otherwise: + - MUST set `total_msat` to the amount it wishes to pay. + - MUST ensure that the total `amount_msat` of the HTLC set which arrives at the payee + is equal to `total_msat`. + - MUST NOT send another HTLC if the total `amount_msat` of the HTLC set is already greater or equal to `total_msat`. + - MUST include `payment_secret`. + - otherwise: + - MUST set `total_msat` equal to `amt_to_forward`. + +The final node: + - MUST fail the HTLC if dictated by Requirements under [Failure Messages](#failure-messages) + - Note: "amount paid" specified there is the `total_msat` field. + - if it does not support `basic_mpp`: + - MUST fail the HTLC if `total_msat` is not exactly equal to `amt_to_forward`. + - otherwise, if it supports `basic_mpp`: + - MUST add it to the HTLC set corresponding to that `payment_hash`. + - SHOULD fail the entire HTLC set if `total_msat` is not the same for + all HTLCs in the set. + - if the total `amount_msat` of this HTLC set equals `total_msat`: + - SHOULD fulfill all HTLCs in the HTLC set + - otherwise, if the total `amount_msat` of this HTLC set is less than + `total_msat`: + - MUST NOT fulfill any HTLCs in the HTLC set + - MUST fail all HTLCs in the HTLC set after some reasonable timeout. + - SHOULD wait for at least 60 seconds after the initial HTLC. + - SHOULD use `mpp_timeout` for the failure message. + - MUST require `payment_secret` for all HTLCs in the set. + - if it fulfills any HTLCs in the HTLC set: + - MUST fulfill the entire HTLC set. + +#### Rationale + +If `basic_mpp` is present it causes a delay to allow other partial +payments to combine. The total amount must be sufficient for the +desired payment, just as it must be for single payments. But this must +be reasonably bounded to avoid a denial-of-service. + +Because invoices do not necessarily specify an amount, and because +payers can add noise to the final amount, the total amount must be +sent explicitly. The requirements allow exceeding this slightly, as +it simplifies adding noise to the amount when splitting, as well as +scenarios in which the senders are genuinely independent (friends +splitting a bill, for example). + +The restriction on sending an HTLC once the set is over the agreed total prevents the preimage being released before all +the partial payments have arrived: that would allow any intermediate +node to immediately claim any outstanding partial payments. + +An implementation may choose not to fulfill an HTLC set which +otherwise meets the amount criterion (eg. some other failure, or +invoice timeout), however if it were to fulfill only some of them, +intermediary nodes could simply claim the remaining ones. + +# Accepting and Forwarding a Payment + +Once a node has decoded the payload it either accepts the payment locally, or forwards it to the peer indicated as the next hop in the payload. + +## Non-strict Forwarding + +A node MAY forward an HTLC along an outgoing channel other than the one +specified by `short_channel_id`, so long as the receiver has the same node +public key intended by `short_channel_id`. Thus, if `short_channel_id` connects +nodes A and B, the HTLC can be forwarded across any channel connecting A and B. +Failure to adhere will result in the receiver being unable to decrypt the next +hop in the onion packet. + +### Rationale + +In the event that two peers have multiple channels, the downstream node will be +able to decrypt the next hop payload regardless of which channel the packet is +sent across. + +Nodes implementing non-strict forwarding are able to make real-time assessments +of channel bandwidths with a particular peer, and use the channel that is +locally-optimal. + +For example, if the channel specified by `short_channel_id` connecting A and B +does not have enough bandwidth at forwarding time, then A is able use a +different channel that does. This can reduce payment latency by preventing the +HTLC from failing due to bandwidth constraints across `short_channel_id`, only +to have the sender attempt the same route differing only in the channel between +A and B. + +Non-strict forwarding allows nodes to make use of private channels connecting +them to the receiving node, even if the channel is not known in the public +channel graph. + +### Recommendation + +Implementations using non-strict forwarding should consider applying the same +fee schedule to all channels with the same peer, as senders are likely to select +the channel which results in the lowest overall cost. Having distinct policies +may result in the forwarding node accepting fees based on the most optimal fee +schedule for the sender, even though they are providing aggregate bandwidth +across all channels with the same peer. + +Alternatively, implementations may choose to apply non-strict forwarding only to +like-policy channels to ensure their expected fee revenue does not deviate by +using an alternate channel. + +## Payload for the Last Node + +When building the route, the origin node MUST use a payload for +the final node with the following values: + +* `payment_secret`: set to the payment secret specified by the recipient (e.g. + `payment_secret` from a [BOLT #11](11-payment-encoding.md) payment invoice) +* `outgoing_cltv_value`: set to the final expiry specified by the recipient (e.g. + `min_final_cltv_expiry` from a [BOLT #11](11-payment-encoding.md) payment invoice) +* `amt_to_forward`: set to the final amount specified by the recipient (e.g. `amount` + from a [BOLT #11](11-payment-encoding.md) payment invoice) + +This allows the final node to check these values and return errors if needed, +but it also eliminates the possibility of probing attacks by the second-to-last +node. Such attacks could, otherwise, attempt to discover if the receiving peer is the +last one by re-sending HTLCs with different amounts/expiries. +The final node will extract its onion payload from the HTLC it has received and +compare its values against those of the HTLC. See the +[Returning Errors](#returning-errors) section below for more details. + +If not for the above, since it need not forward payments, the final node could +simply discard its payload. + +# Shared Secret + +The origin node establishes a shared secret with each hop along the route using +Elliptic-curve Diffie-Hellman between the sender's ephemeral key at that hop and +the hop's node ID key. The resulting curve point is serialized to the +compressed format and hashed using `SHA256`. The hash output is used +as the 32-byte shared secret. + +Elliptic-curve Diffie-Hellman (ECDH) is an operation on an EC private key and +an EC public key that outputs a curve point. For this protocol, the ECDH +variant implemented in `libsecp256k1` is used, which is defined over the +`secp256k1` elliptic curve. During packet construction, the sender uses the +ephemeral private key and the hop's public key as inputs to ECDH, whereas +during packet forwarding, the hop uses the ephemeral public key and its own +node ID private key. Because of the properties of ECDH, they will both derive +the same value. + +# Blinding Ephemeral Keys + +In order to ensure multiple hops along the route cannot be linked by the +ephemeral public keys they see, the key is blinded at each hop. The blinding is +done in a deterministic way that allows the sender to compute the +corresponding blinded private keys during packet construction. + +The blinding of an EC public key is a single scalar multiplication of +the EC point representing the public key with a 32-byte blinding factor. Due to +the commutative property of scalar multiplication, the blinded private key is +the multiplicative product of the input's corresponding private key with the +same blinding factor. + +The blinding factor itself is computed as a function of the ephemeral public key +and the 32-byte shared secret. Concretely, it is the `SHA256` hash value of the +concatenation of the public key serialized in its compressed format and the +shared secret. + +# Packet Construction + +In the following example, it's assumed that a _sending node_ (origin node), +`n_0`, wants to route a packet to a _receiving node_ (final node), `n_r`. +First, the sender computes a route `{n_0, n_1, ..., n_{r-1}, n_r}`, where `n_0` +is the sender itself and `n_r` is the final recipient. All nodes `n_i` and +`n_{i+1}` MUST be peers in the overlay network route. The sender then gathers the +public keys for `n_1` to `n_r` and generates a random 32-byte `sessionkey`. +Optionally, the sender may pass in _associated data_, i.e. data that the +packet commits to but that is not included in the packet itself. Associated +data will be included in the HMACs and must match the associated data provided +during integrity verification at each hop. + +To construct the onion, the sender initializes the ephemeral private key for the +first hop `ek_1` to the `sessionkey` and derives from it the corresponding +ephemeral public key `epk_1` by multiplying with the `secp256k1` base point. For +each of the `k` hops along the route, the sender then iteratively computes the +shared secret `ss_k` and ephemeral key for the next hop `ek_{k+1}` as follows: + + - The sender executes ECDH with the hop's public key and the ephemeral private + key to obtain a curve point, which is hashed using `SHA256` to produce the + shared secret `ss_k`. + - The blinding factor is the `SHA256` hash of the concatenation between the + ephemeral public key `epk_k` and the shared secret `ss_k`. + - The ephemeral private key for the next hop `ek_{k+1}` is computed by + multiplying the current ephemeral private key `ek_k` by the blinding factor. + - The ephemeral public key for the next hop `epk_{k+1}` is derived from the + ephemeral private key `ek_{k+1}` by multiplying with the base point. + +Once the sender has all the required information above, it can construct the +packet. Constructing a packet routed over `r` hops requires `r` 32-byte +ephemeral public keys, `r` 32-byte shared secrets, `r` 32-byte blinding factors, +and `r` variable length `hop_payload` payloads. +The construction returns a single 1366-byte packet along with the first receiving peer's address. + +The packet construction is performed in the reverse order of the route, i.e. +the last hop's operations are applied first. + +The packet is initialized with 1300 _random_ bytes derived from a CSPRNG +(ChaCha20). The _pad_ key referenced above is used to extract additional random +bytes from a ChaCha20 stream, using it as a CSPRNG for this purpose. Once the +`paddingKey` has been obtained, ChaCha20 is used with an all zero nonce, to +generate 1300 random bytes. Those random bytes are then used as the starting +state of the mix-header to be created. + +A filler is generated (see [Filler Generation](#filler-generation)) using the +shared secret. + +For each hop in the route, in reverse order, the sender applies the +following operations: + + - The _rho_-key and _mu_-key are generated using the hop's shared secret. + - `shift_size` is defined as the length of the `hop_payload` plus the bigsize encoding of the length and the length of that HMAC. Thus if the payload length is `l` then the `shift_size` is `1 + l + 32` for `l < 253`, otherwise `3 + l + 32` due to the bigsize encoding of `l`. + - The `hop_payload` field is right-shifted by `shift_size` bytes, discarding the last `shift_size` + bytes that exceed its 1300-byte size. + - The bigsize-serialized length, serialized `hop_payload` and `hmac` are copied into the following `shift_size` bytes. + - The _rho_-key is used to generate 1300 bytes of pseudo-random byte stream + which is then applied, with `XOR`, to the `hop_payloads` field. + - If this is the last hop, i.e. the first iteration, then the tail of the + `hop_payloads` field is overwritten with the routing information `filler`. + - The next HMAC is computed (with the _mu_-key as HMAC-key) over the + concatenated `hop_payloads` and associated data. + +The resulting final HMAC value is the HMAC that will be used by the first +receiving peer in the route. + +The packet generation returns a serialized packet that contains the `version` +byte, the ephemeral pubkey for the first hop, the HMAC for the first hop, and +the obfuscated `hop_payloads`. + +The following Go code is an example implementation of the packet construction: + +```Go +func NewOnionPacket(paymentPath []*btcec.PublicKey, sessionKey *btcec.PrivateKey, + hopsData []HopData, assocData []byte) (*OnionPacket, error) { + + numHops := len(paymentPath) + hopSharedSecrets := make([][sha256.Size]byte, numHops) + + // Initialize ephemeral key for the first hop to the session key. + var ephemeralKey big.Int + ephemeralKey.Set(sessionKey.D) + + for i := 0; i < numHops; i++ { + // Perform ECDH and hash the result. + ecdhResult := scalarMult(paymentPath[i], ephemeralKey) + hopSharedSecrets[i] = sha256.Sum256(ecdhResult.SerializeCompressed()) + + // Derive ephemeral public key from private key. + ephemeralPrivKey := btcec.PrivKeyFromBytes(btcec.S256(), ephemeralKey.Bytes()) + ephemeralPubKey := ephemeralPrivKey.PubKey() + + // Compute blinding factor. + sha := sha256.New() + sha.Write(ephemeralPubKey.SerializeCompressed()) + sha.Write(hopSharedSecrets[i]) + + var blindingFactor big.Int + blindingFactor.SetBytes(sha.Sum(nil)) + + // Blind ephemeral key for next hop. + ephemeralKey.Mul(&ephemeralKey, &blindingFactor) + ephemeralKey.Mod(&ephemeralKey, btcec.S256().Params().N) + } + + // Generate the padding, called "filler strings" in the paper. + filler := generateHeaderPadding("rho", numHops, hopDataSize, hopSharedSecrets) + + // Allocate and initialize fields to zero-filled slices + var mixHeader [routingInfoSize]byte + var nextHmac [hmacSize]byte + + // Our starting packet needs to be filled out with random bytes, we + // generate some determinstically using the session private key. + paddingKey := generateKey("pad", sessionKey.Serialize() + paddingBytes := generateCipherStream(paddingKey, routingInfoSize) + copy(mixHeader[:], paddingBytes) + + // Compute the routing information for each hop along with a + // MAC of the routing information using the shared key for that hop. + for i := numHops - 1; i >= 0; i-- { + rhoKey := generateKey("rho", hopSharedSecrets[i]) + muKey := generateKey("mu", hopSharedSecrets[i]) + + hopsData[i].HMAC = nextHmac + + // Shift and obfuscate routing information + streamBytes := generateCipherStream(rhoKey, numStreamBytes) + + rightShift(mixHeader[:], hopDataSize) + buf := &bytes.Buffer{} + hopsData[i].Encode(buf) + copy(mixHeader[:], buf.Bytes()) + xor(mixHeader[:], mixHeader[:], streamBytes[:routingInfoSize]) + + // These need to be overwritten, so every node generates a correct padding + if i == numHops-1 { + copy(mixHeader[len(mixHeader)-len(filler):], filler) + } + + packet := append(mixHeader[:], assocData...) + nextHmac = calcMac(muKey, packet) + } + + packet := &OnionPacket{ + Version: 0x00, + EphemeralKey: sessionKey.PubKey(), + RoutingInfo: mixHeader, + HeaderMAC: nextHmac, + } + return packet, nil +} +``` + +# Packet Forwarding + +This specification is limited to `version` `0` packets; the structure +of future versions may change. + +Upon receiving a packet, a processing node compares the version byte of the +packet with its own supported versions and aborts the connection if the packet +specifies a version number that it doesn't support. +For packets with supported version numbers, the processing node first parses the +packet into its individual fields. + +Next, the processing node computes the shared secret using the private key +corresponding to its own public key and the ephemeral key from the packet, as +described in [Shared Secret](#shared-secret). + +The above requirements prevent any hop along the route from retrying a payment +multiple times, in an attempt to track a payment's progress via traffic +analysis. Note that disabling such probing could be accomplished using a log of +previous shared secrets or HMACs, which could be forgotten once the HTLC would +not be accepted anyway (i.e. after `outgoing_cltv_value` has passed). Such a log +may use a probabilistic data structure, but it MUST rate-limit commitments as +necessary, in order to constrain the worst-case storage requirements or false +positives of this log. + +Next, the processing node uses the shared secret to compute a _mu_-key, which it +in turn uses to compute the HMAC of the `hop_payloads`. The resulting HMAC is then +compared against the packet's HMAC. + +Comparison of the computed HMAC and the packet's HMAC MUST be +time-constant to avoid information leaks. + +At this point, the processing node can generate a _rho_-key and a _gamma_-key. + +The routing information is then deobfuscated, and the information about the +next hop is extracted. +To do so, the processing node copies the `hop_payloads` field, appends 1300 `0x00`-bytes, +generates `2*1300` pseudo-random bytes (using the _rho_-key), and applies the result, using `XOR`, to the copy of the `hop_payloads`. +The first few bytes correspond to the bigsize-encoded length `l` of the `hop_payload`, followed by `l` bytes of the resulting routing information become the `hop_payload`, and the 32 byte HMAC. +The next 1300 bytes are the `hop_payloads` for the outgoing packet. + +A special `hmac` value of 32 `0x00`-bytes indicates that the currently processing hop is the intended recipient and that the packet should not be forwarded. + +If the HMAC does not indicate route termination, and if the next hop is a peer of the +processing node; then the new packet is assembled. Packet assembly is accomplished +by blinding the ephemeral key with the processing node's public key, along with the +shared secret, and by serializing the `hop_payloads`. +The resulting packet is then forwarded to the addressed peer. + +## Requirements + +The processing node: + - if the ephemeral public key is NOT on the `secp256k1` curve: + - MUST abort processing the packet. + - MUST report a route failure to the origin node. + - if the packet has previously been forwarded or locally redeemed, i.e. the + packet contains duplicate routing information to a previously received packet: + - if preimage is known: + - MAY immediately redeem the HTLC using the preimage. + - otherwise: + - MUST abort processing and report a route failure. + - if the computed HMAC and the packet's HMAC differ: + - MUST abort processing. + - MUST report a route failure. + - if the `realm` is unknown: + - MUST drop the packet. + - MUST signal a route failure. + - MUST address the packet to another peer that is its direct neighbor. + - if the processing node does not have a peer with the matching address: + - MUST drop the packet. + - MUST signal a route failure. + + +# Filler Generation + +Upon receiving a packet, the processing node extracts the information destined +for it from the route information and the per-hop payload. +The extraction is done by deobfuscating and left-shifting the field. +This would make the field shorter at each hop, allowing an attacker to deduce the +route length. For this reason, the field is pre-padded before forwarding. +Since the padding is part of the HMAC, the origin node will have to pre-generate an +identical padding (to that which each hop will generate) in order to compute the +HMACs correctly for each hop. +The filler is also used to pad the field-length, in the case that the selected +route is shorter than 1300 bytes. + +Before deobfuscating the `hop_payloads`, the processing node pads it with 1300 +`0x00`-bytes, such that the total length is `2*1300`. +It then generates the pseudo-random byte stream, of matching length, and applies +it with `XOR` to the `hop_payloads`. +This deobfuscates the information destined for it, while simultaneously +obfuscating the added `0x00`-bytes at the end. + +In order to compute the correct HMAC, the origin node has to pre-generate the +`hop_payloads` for each hop, including the incrementally obfuscated padding added +by each hop. This incrementally obfuscated padding is referred to as the +`filler`. + +The following example code shows how the filler is generated in Go: + +```Go +func generateFiller(key string, numHops int, hopSize int, sharedSecrets [][sharedSecretSize]byte) []byte { + fillerSize := uint((numMaxHops + 1) * hopSize) + filler := make([]byte, fillerSize) + + // The last hop does not obfuscate, it's not forwarding anymore. + for i := 0; i < numHops-1; i++ { + + // Left-shift the field + copy(filler[:], filler[hopSize:]) + + // Zero-fill the last hop + copy(filler[len(filler)-hopSize:], bytes.Repeat([]byte{0x00}, hopSize)) + + // Generate pseudo-random byte stream + streamKey := generateKey(key, sharedSecrets[i]) + streamBytes := generateCipherStream(streamKey, fillerSize) + + // Obfuscate + xor(filler, filler, streamBytes) + } + + // Cut filler down to the correct length (numHops+1)*hopSize + // bytes will be prepended by the packet generation. + return filler[(numMaxHops-numHops+2)*hopSize:] +} +``` + +Note that this example implementation is for demonstration purposes only; the +`filler` can be generated much more efficiently. +The last hop need not obfuscate the `filler`, since it won't forward the packet +any further and thus need not extract an HMAC either. + +# Returning Errors + +The onion routing protocol includes a simple mechanism for returning encrypted +error messages to the origin node. +The returned error messages may be failures reported by any hop, including the +final node. +The format of the forward packet is not usable for the return path, since no hop +besides the origin has access to the information required for its generation. +Note that these error messages are not reliable, as they are not placed on-chain +due to the possibility of hop failure. + +Intermediate hops store the shared secret from the forward path and reuse it to +obfuscate any corresponding return packet during each hop. +In addition, each node locally stores data regarding its own sending peer in the +route, so it knows where to return-forward any eventual return packets. +The node generating the error message (_erring node_) builds a return packet +consisting of the following fields: + +1. data: + * [`32*byte`:`hmac`] + * [`u16`:`failure_len`] + * [`failure_len*byte`:`failuremsg`] + * [`u16`:`pad_len`] + * [`pad_len*byte`:`pad`] + +Where `hmac` is an HMAC authenticating the remainder of the packet, with a key +generated using the above process, with key type `um`, `failuremsg` as defined +below, and `pad` as the extra bytes used to conceal length. + +The erring node then generates a new key, using the key type `ammag`. +This key is then used to generate a pseudo-random stream, which is in turn +applied to the packet using `XOR`. + +The obfuscation step is repeated by every hop along the return path. +Upon receiving a return packet, each hop generates its `ammag`, generates the +pseudo-random byte stream, and applies the result to the return packet before +return-forwarding it. + +The origin node is able to detect that it's the intended final recipient of the +return message, because of course, it was the originator of the corresponding +forward packet. +When an origin node receives an error message matching a transfer it initiated +(i.e. it cannot return-forward the error any further) it generates the `ammag` +and `um` keys for each hop in the route. +It then iteratively decrypts the error message, using each hop's `ammag` +key, and computes the HMAC, using each hop's `um` key. +The origin node can detect the sender of the error message by matching the +`hmac` field with the computed HMAC. + +The association between the forward and return packets is handled outside of +this onion routing protocol, e.g. via association with an HTLC in a payment +channel. + +### Requirements + +The _erring node_: + - SHOULD set `pad` such that the `failure_len` plus `pad_len` is equal to 256. + - Note: this value is 118 bytes longer than the longest currently-defined + message. + +The _origin node_: + - once the return message has been decrypted: + - SHOULD store a copy of the message. + - SHOULD continue decrypting, until the loop has been repeated 20 times. + - SHOULD use constant `ammag` and `um` keys to obfuscate the route length. + +## Failure Messages + +The failure message encapsulated in `failuremsg` has an identical format as +a normal message: a 2-byte type `failure_code` followed by data applicable +to that type. Below is a list of the currently supported `failure_code` +values, followed by their use case requirements. + +Notice that the `failure_code`s are not of the same type as other message types, +defined in other BOLTs, as they are not sent directly on the transport layer +but are instead wrapped inside return packets. +The numeric values for the `failure_code` may therefore reuse values, that are +also assigned to other message types, without any danger of causing collisions. + +The top byte of `failure_code` can be read as a set of flags: +* 0x8000 (BADONION): unparsable onion encrypted by sending peer +* 0x4000 (PERM): permanent failure (otherwise transient) +* 0x2000 (NODE): node failure (otherwise channel) +* 0x1000 (UPDATE): new channel update enclosed + +Please note that the `channel_update` field is mandatory in messages whose +`failure_code` includes the `UPDATE` flag. + +The following `failure_code`s are defined: + +1. type: PERM|1 (`invalid_realm`) + +The `realm` byte was not understood by the processing node. + +1. type: NODE|2 (`temporary_node_failure`) + +General temporary failure of the processing node. + +1. type: PERM|NODE|2 (`permanent_node_failure`) + +General permanent failure of the processing node. + +1. type: PERM|NODE|3 (`required_node_feature_missing`) + +The processing node has a required feature which was not in this onion. + +1. type: BADONION|PERM|4 (`invalid_onion_version`) +2. data: + * [`sha256`:`sha256_of_onion`] + +The `version` byte was not understood by the processing node. + +1. type: BADONION|PERM|5 (`invalid_onion_hmac`) +2. data: + * [`sha256`:`sha256_of_onion`] + +The HMAC of the onion was incorrect when it reached the processing node. + +1. type: BADONION|PERM|6 (`invalid_onion_key`) +2. data: + * [`sha256`:`sha256_of_onion`] + +The ephemeral key was unparsable by the processing node. + +1. type: UPDATE|7 (`temporary_channel_failure`) +2. data: + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The channel from the processing node was unable to handle this HTLC, +but may be able to handle it, or others, later. + +1. type: PERM|8 (`permanent_channel_failure`) + +The channel from the processing node is unable to handle any HTLCs. + +1. type: PERM|9 (`required_channel_feature_missing`) + +The channel from the processing node requires features not present in +the onion. + +1. type: PERM|10 (`unknown_next_peer`) + +The onion specified a `short_channel_id` which doesn't match any +leading from the processing node. + +1. type: UPDATE|11 (`amount_below_minimum`) +2. data: + * [`u64`:`htlc_msat`] + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The HTLC amount was below the `htlc_minimum_msat` of the channel from +the processing node. + +1. type: UPDATE|12 (`fee_insufficient`) +2. data: + * [`u64`:`htlc_msat`] + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The fee amount was below that required by the channel from the +processing node. + +1. type: UPDATE|13 (`incorrect_cltv_expiry`) +2. data: + * [`u32`:`cltv_expiry`] + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The `cltv_expiry` does not comply with the `cltv_expiry_delta` required by +the channel from the processing node: it does not satisfy the following +requirement: + + cltv_expiry - cltv_expiry_delta >= outgoing_cltv_value + +1. type: UPDATE|14 (`expiry_too_soon`) +2. data: + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The CLTV expiry is too close to the current block height for safe +handling by the processing node. + +1. type: PERM|15 (`incorrect_or_unknown_payment_details`) +2. data: + * [`u64`:`htlc_msat`] + * [`u32`:`height`] + +The `payment_hash` is unknown to the final node, the `payment_secret` doesn't +match the `payment_hash`, the amount for that `payment_hash` is incorrect or +the CLTV expiry of the htlc is too close to the current block height for safe +handling. + +The `htlc_msat` parameter is superfluous, but left in for backwards +compatibility. The value of `htlc_msat` always matches the amount specified in +the final hop onion payload. It therefore does not have any informative value to +the sender. A penultimate hop sending a different amount or expiry for the htlc +is handled through `final_incorrect_cltv_expiry` and +`final_incorrect_htlc_amount`. + +The `height` parameter is set by the final node to the best known block height +at the time of receiving the htlc. This can be used by the sender to distinguish +between sending a payment with the wrong final CLTV expiry and an intermediate +hop delaying the payment so that the receiver's invoice CLTV delta requirement +is no longer met. + +Note: Originally PERM|16 (`incorrect_payment_amount`) and 17 +(`final_expiry_too_soon`) were used to differentiate incorrect htlc parameters +from unknown payment hash. Sadly, sending this response allows for probing +attacks whereby a node which receives an HTLC for forwarding can check guesses +as to its final destination by sending payments with the same hash but much +lower values or expiry heights to potential destinations and check the response. +Care must be taken by implementations to differentiate the previously +non-permanent case for `final_expiry_too_soon` (17) from the other, permanent +failures now represented by `incorrect_or_unknown_payment_details` (PERM|15). + +1. type: 18 (`final_incorrect_cltv_expiry`) +2. data: + * [`u32`:`cltv_expiry`] + +The CLTV expiry in the HTLC doesn't match the value in the onion. + +1. type: 19 (`final_incorrect_htlc_amount`) +2. data: + * [`u64`:`incoming_htlc_amt`] + +The amount in the HTLC doesn't match the value in the onion. + +1. type: UPDATE|20 (`channel_disabled`) +2. data: + * [`u16`: `flags`] + * [`u16`:`len`] + * [`len*byte`:`channel_update`] + +The channel from the processing node has been disabled. + +1. type: 21 (`expiry_too_far`) + +The CLTV expiry in the HTLC is too far in the future. + +1. type: PERM|22 (`invalid_onion_payload`) +2. data: + * [`bigsize`:`type`] + * [`u16`:`offset`] + +The decrypted onion per-hop payload was not understood by the processing node +or is incomplete. If the failure can be narrowed down to a specific tlv type in +the payload, the erring node may include that `type` and its byte `offset` in +the decrypted byte stream. + +1. type: 23 (`mpp_timeout`) + +The complete amount of the multi-part payment was not received within a +reasonable time. + +### Requirements + +An _erring node_: + - MUST select one of the above error codes when creating an error message. + - MUST include the appropriate data for that particular error type. + - if there is more than one error: + - SHOULD select the first error it encounters from the list above. + +Any _erring node_ MAY: + - if the `realm` byte is unknown: + - return an `invalid_realm` error. + - if the per-hop payload in the onion is invalid (e.g. it is not a valid tlv stream) + or is missing required information (e.g. the amount was not specified): + - return an `invalid_onion_payload` error. + - if an otherwise unspecified transient error occurs for the entire node: + - return a `temporary_node_failure` error. + - if an otherwise unspecified permanent error occurs for the entire node: + - return a `permanent_node_failure` error. + - if a node has requirements advertised in its `node_announcement` `features`, + which were NOT included in the onion: + - return a `required_node_feature_missing` error. + +A _forwarding node_ MAY, but a _final node_ MUST NOT: + - if the onion `version` byte is unknown: + - return an `invalid_onion_version` error. + - if the onion HMAC is incorrect: + - return an `invalid_onion_hmac` error. + - if the ephemeral key in the onion is unparsable: + - return an `invalid_onion_key` error. + - if during forwarding to its receiving peer, an otherwise unspecified, + transient error occurs in the outgoing channel (e.g. channel capacity reached, + too many in-flight HTLCs, etc.): + - return a `temporary_channel_failure` error. + - if an otherwise unspecified, permanent error occurs during forwarding to its + receiving peer (e.g. channel recently closed): + - return a `permanent_channel_failure` error. + - if the outgoing channel has requirements advertised in its + `channel_announcement`'s `features`, which were NOT included in the onion: + - return a `required_channel_feature_missing` error. + - if the receiving peer specified by the onion is NOT known: + - return an `unknown_next_peer` error. + - if the HTLC amount is less than the currently specified minimum amount: + - report the amount of the outgoing HTLC and the current channel setting for + the outgoing channel. + - return an `amount_below_minimum` error. + - if the HTLC does NOT pay a sufficient fee: + - report the amount of the incoming HTLC and the current channel setting for + the outgoing channel. + - return a `fee_insufficient` error. + - if the incoming `cltv_expiry` minus the `outgoing_cltv_value` is below the + `cltv_expiry_delta` for the outgoing channel: + - report the `cltv_expiry` of the outgoing HTLC and the current channel setting for the outgoing + channel. + - return an `incorrect_cltv_expiry` error. + - if the `cltv_expiry` is unreasonably near the present: + - report the current channel setting for the outgoing channel. + - return an `expiry_too_soon` error. + - if the `cltv_expiry` is unreasonably far in the future: + - return an `expiry_too_far` error. + - if the channel is disabled: + - report the current channel setting for the outgoing channel. + - return a `channel_disabled` error. + +An _intermediate hop_ MUST NOT, but the _final node_: + - if the payment hash has already been paid: + - MAY treat the payment hash as unknown. + - MAY succeed in accepting the HTLC. + - if the `payment_secret` doesn't match the expected value for that `payment_hash`, + or the `payment_secret` is required and is not present: + - MUST fail the HTLC. + - MUST return an `incorrect_or_unknown_payment_details` error. + - if the amount paid is less than the amount expected: + - MUST fail the HTLC. + - MUST return an `incorrect_or_unknown_payment_details` error. + - if the payment hash is unknown: + - MUST fail the HTLC. + - MUST return an `incorrect_or_unknown_payment_details` error. + - if the amount paid is more than twice the amount expected: + - SHOULD fail the HTLC. + - SHOULD return an `incorrect_or_unknown_payment_details` error. + - Note: this allows the origin node to reduce information leakage by + altering the amount while not allowing for accidental gross overpayment. + - if the `cltv_expiry` value is unreasonably near the present: + - MUST fail the HTLC. + - MUST return an `incorrect_or_unknown_payment_details` error. + - if the `outgoing_cltv_value` does NOT correspond with the `cltv_expiry` from + the final node's HTLC: + - MUST return `final_incorrect_cltv_expiry` error. + - if the `amt_to_forward` does NOT correspond with the `incoming_htlc_amt` from the + final node's HTLC: + - MUST return a `final_incorrect_htlc_amount` error. + +## Receiving Failure Codes + +### Requirements + +The _origin node_: + - MUST ignore any extra bytes in `failuremsg`. + - if the _final node_ is returning the error: + - if the PERM bit is set: + - SHOULD fail the payment. + - otherwise: + - if the error code is understood and valid: + - MAY retry the payment. In particular, `final_expiry_too_soon` can + occur if the block height has changed since sending, and in this case + `temporary_node_failure` could resolve within a few seconds. + - otherwise, an _intermediate hop_ is returning the error: + - if the NODE bit is set: + - SHOULD remove all channels connected with the erring node from + consideration. + - if the PERM bit is NOT set: + - SHOULD restore the channels as it receives new `channel_update`s. + - otherwise: + - if UPDATE is set, AND the `channel_update` is valid and more recent + than the `channel_update` used to send the payment: + - if `channel_update` should NOT have caused the failure: + - MAY treat the `channel_update` as invalid. + - otherwise: + - SHOULD apply the `channel_update`. + - MAY queue the `channel_update` for broadcast. + - otherwise: + - SHOULD eliminate the channel outgoing from the erring node from + consideration. + - if the PERM bit is NOT set: + - SHOULD restore the channel as it receives new `channel_update`s. + - SHOULD then retry routing and sending the payment. + - MAY use the data specified in the various failure types for debugging + purposes. + +# Test Vector + +## Returning Errors + +The test vectors use the following parameters: + + pubkey[0] = 0x02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619 + pubkey[1] = 0x0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c + pubkey[2] = 0x027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007 + pubkey[3] = 0x032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991 + pubkey[4] = 0x02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145 + + nhops = 5/20 + sessionkey = 0x4141414141414141414141414141414141414141414141414141414141414141 + associated data = 0x4242424242424242424242424242424242424242424242424242424242424242 + +The following is an in-depth trace of an example of error message creation: + + # node 4 is returning an error + failure_message = 2002 + # creating error message + shared_secret = b5756b9b542727dbafc6765a49488b023a725d631af688fc031217e90770c328 + payload = 0002200200fe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + um_key = 4da7f2923edce6c2d85987d1d9fa6d88023e6c3a9c3d20f07d3b10b61a78d646 + raw_error_packet = 4c2fc8bc08510334b6833ad9c3e79cd1b52ae59dfe5c2a4b23ead50f09f7ee0b0002200200fe0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + # forwarding error packet + shared_secret = b5756b9b542727dbafc6765a49488b023a725d631af688fc031217e90770c328 + ammag_key = 2f36bb8822e1f0d04c27b7d8bb7d7dd586e032a3218b8d414afbba6f169a4d68 + stream = e9c975b07c9a374ba64fd9be3aae955e917d34d1fa33f2e90f53bbf4394713c6a8c9b16ab5f12fd45edd73c1b0c8b33002df376801ff58aaa94000bf8a86f92620f343baef38a580102395ae3abf9128d1047a0736ff9b83d456740ebbb4aeb3aa9737f18fb4afb4aa074fb26c4d702f42968888550a3bded8c05247e045b866baef0499f079fdaeef6538f31d44deafffdfd3afa2fb4ca9082b8f1c465371a9894dd8c243fb4847e004f5256b3e90e2edde4c9fb3082ddfe4d1e734cacd96ef0706bf63c9984e22dc98851bcccd1c3494351feb458c9c6af41c0044bea3c47552b1d992ae542b17a2d0bba1a096c78d169034ecb55b6e3a7263c26017f033031228833c1daefc0dedb8cf7c3e37c9c37ebfe42f3225c326e8bcfd338804c145b16e34e4 + error packet for node 4: a5e6bd0c74cb347f10cce367f949098f2457d14c046fd8a22cb96efb30b0fdcda8cb9168b50f2fd45edd73c1b0c8b33002df376801ff58aaa94000bf8a86f92620f343baef38a580102395ae3abf9128d1047a0736ff9b83d456740ebbb4aeb3aa9737f18fb4afb4aa074fb26c4d702f42968888550a3bded8c05247e045b866baef0499f079fdaeef6538f31d44deafffdfd3afa2fb4ca9082b8f1c465371a9894dd8c243fb4847e004f5256b3e90e2edde4c9fb3082ddfe4d1e734cacd96ef0706bf63c9984e22dc98851bcccd1c3494351feb458c9c6af41c0044bea3c47552b1d992ae542b17a2d0bba1a096c78d169034ecb55b6e3a7263c26017f033031228833c1daefc0dedb8cf7c3e37c9c37ebfe42f3225c326e8bcfd338804c145b16e34e4 + # forwarding error packet + shared_secret = 21e13c2d7cfe7e18836df50872466117a295783ab8aab0e7ecc8c725503ad02d + ammag_key = cd9ac0e09064f039fa43a31dea05f5fe5f6443d40a98be4071af4a9d704be5ad + stream = 617ca1e4624bc3f04fece3aa5a2b615110f421ec62408d16c48ea6c1b7c33fe7084a2bd9d4652fc5068e5052bf6d0acae2176018a3d8c75f37842712913900263cff92f39f3c18aa1f4b20a93e70fc429af7b2b1967ca81a761d40582daf0eb49cef66e3d6fbca0218d3022d32e994b41c884a27c28685ef1eb14603ea80a204b2f2f474b6ad5e71c6389843e3611ebeafc62390b717ca53b3670a33c517ef28a659c251d648bf4c966a4ef187113ec9848bf110816061ca4f2f68e76ceb88bd6208376460b916fb2ddeb77a65e8f88b2e71a2cbf4ea4958041d71c17d05680c051c3676fb0dc8108e5d78fb1e2c44d79a202e9d14071d536371ad47c39a05159e8d6c41d17a1e858faaaf572623aa23a38ffc73a4114cb1ab1cd7f906c6bd4e21b29694 + error packet for node 3: c49a1ce81680f78f5f2000cda36268de34a3f0a0662f55b4e837c83a8773c22aa081bab1616a0011585323930fa5b9fae0c85770a2279ff59ec427ad1bbff9001c0cd1497004bd2a0f68b50704cf6d6a4bf3c8b6a0833399a24b3456961ba00736785112594f65b6b2d44d9f5ea4e49b5e1ec2af978cbe31c67114440ac51a62081df0ed46d4a3df295da0b0fe25c0115019f03f15ec86fabb4c852f83449e812f141a9395b3f70b766ebbd4ec2fae2b6955bd8f32684c15abfe8fd3a6261e52650e8807a92158d9f1463261a925e4bfba44bd20b166d532f0017185c3a6ac7957adefe45559e3072c8dc35abeba835a8cb01a71a15c736911126f27d46a36168ca5ef7dccd4e2886212602b181463e0dd30185c96348f9743a02aca8ec27c0b90dca270 + forwarding error packet + shared_secret = 3a6b412548762f0dbccce5c7ae7bb8147d1caf9b5471c34120b30bc9c04891cc + ammag_key = 1bf08df8628d452141d56adfd1b25c1530d7921c23cecfc749ac03a9b694b0d3 + stream = 6149f48b5a7e8f3d6f5d870b7a698e204cf64452aab4484ff1dee671fe63fd4b5f1b78ee2047dfa61e3d576b149bedaf83058f85f06a3172a3223ad6c4732d96b32955da7d2feb4140e58d86fc0f2eb5d9d1878e6f8a7f65ab9212030e8e915573ebbd7f35e1a430890be7e67c3fb4bbf2def662fa625421e7b411c29ebe81ec67b77355596b05cc155755664e59c16e21410aabe53e80404a615f44ebb31b365ca77a6e91241667b26c6cad24fb2324cf64e8b9dd6e2ce65f1f098cfd1ef41ba2d4c7def0ff165a0e7c84e7597c40e3dffe97d417c144545a0e38ee33ebaae12cc0c14650e453d46bfc48c0514f354773435ee89b7b2810606eb73262c77a1d67f3633705178d79a1078c3a01b5fadc9651feb63603d19decd3a00c1f69af2dab259593 + error packet for node 2: a5d3e8634cfe78b2307d87c6d90be6fe7855b4f2cc9b1dfb19e92e4b79103f61ff9ac25f412ddfb7466e74f81b3e545563cdd8f5524dae873de61d7bdfccd496af2584930d2b566b4f8d3881f8c043df92224f38cf094cfc09d92655989531524593ec6d6caec1863bdfaa79229b5020acc034cd6deeea1021c50586947b9b8e6faa83b81fbfa6133c0af5d6b07c017f7158fa94f0d206baf12dda6b68f785b773b360fd0497e16cc402d779c8d48d0fa6315536ef0660f3f4e1865f5b38ea49c7da4fd959de4e83ff3ab686f059a45c65ba2af4a6a79166aa0f496bf04d06987b6d2ea205bdb0d347718b9aeff5b61dfff344993a275b79717cd815b6ad4c0beb568c4ac9c36ff1c315ec1119a1993c4b61e6eaa0375e0aaf738ac691abd3263bf937e3 + # forwarding error packet + shared_secret = a6519e98832a0b179f62123b3567c106db99ee37bef036e783263602f3488fae + ammag_key = 59ee5867c5c151daa31e36ee42530f429c433836286e63744f2020b980302564 + stream = 0f10c86f05968dd91188b998ee45dcddfbf89fe9a99aa6375c42ed5520a257e048456fe417c15219ce39d921555956ae2ff795177c63c819233f3bcb9b8b28e5ac6e33a3f9b87ca62dff43f4cc4a2755830a3b7e98c326b278e2bd31f4a9973ee99121c62873f5bfb2d159d3d48c5851e3b341f9f6634f51939188c3b9ff45feeb11160bb39ce3332168b8e744a92107db575ace7866e4b8f390f1edc4acd726ed106555900a0832575c3a7ad11bb1fe388ff32b99bcf2a0d0767a83cf293a220a983ad014d404bfa20022d8b369fe06f7ecc9c74751dcda0ff39d8bca74bf9956745ba4e5d299e0da8f68a9f660040beac03e795a046640cf8271307a8b64780b0588422f5a60ed7e36d60417562938b400802dac5f87f267204b6d5bcfd8a05b221ec2 + error packet for node 1: aac3200c4968f56b21f53e5e374e3a2383ad2b1b6501bbcc45abc31e59b26881b7dfadbb56ec8dae8857add94e6702fb4c3a4de22e2e669e1ed926b04447fc73034bb730f4932acd62727b75348a648a1128744657ca6a4e713b9b646c3ca66cac02cdab44dd3439890ef3aaf61708714f7375349b8da541b2548d452d84de7084bb95b3ac2345201d624d31f4d52078aa0fa05a88b4e20202bd2b86ac5b52919ea305a8949de95e935eed0319cf3cf19ebea61d76ba92532497fcdc9411d06bcd4275094d0a4a3c5d3a945e43305a5a9256e333e1f64dbca5fcd4e03a39b9012d197506e06f29339dfee3331995b21615337ae060233d39befea925cc262873e0530408e6990f1cbd233a150ef7b004ff6166c70c68d9f8c853c1abca640b8660db2921 + # forwarding error packet + shared_secret = 53eb63ea8a3fec3b3cd433b85cd62a4b145e1dda09391b348c4e1cd36a03ea66 + ammag_key = 3761ba4d3e726d8abb16cba5950ee976b84937b61b7ad09e741724d7dee12eb5 + stream = 3699fd352a948a05f604763c0bca2968d5eaca2b0118602e52e59121f050936c8dd90c24df7dc8cf8f1665e39a6c75e9e2c0900ea245c9ed3b0008148e0ae18bbfaea0c711d67eade980c6f5452e91a06b070bbde68b5494a92575c114660fb53cf04bf686e67ffa4a0f5ae41a59a39a8515cb686db553d25e71e7a97cc2febcac55df2711b6209c502b2f8827b13d3ad2f491c45a0cafe7b4d8d8810e805dee25d676ce92e0619b9c206f922132d806138713a8f69589c18c3fdc5acee41c1234b17ecab96b8c56a46787bba2c062468a13919afc18513835b472a79b2c35f9a91f38eb3b9e998b1000cc4a0dbd62ac1a5cc8102e373526d7e8f3c3a1b4bfb2f8a3947fe350cb89f73aa1bb054edfa9895c0fc971c2b5056dc8665902b51fced6dff80c + error packet for node 0: 9c5add3963fc7f6ed7f148623c84134b5647e1306419dbe2174e523fa9e2fbed3a06a19f899145610741c83ad40b7712aefaddec8c6baf7325d92ea4ca4d1df8bce517f7e54554608bf2bd8071a4f52a7a2f7ffbb1413edad81eeea5785aa9d990f2865dc23b4bc3c301a94eec4eabebca66be5cf638f693ec256aec514620cc28ee4a94bd9565bc4d4962b9d3641d4278fb319ed2b84de5b665f307a2db0f7fbb757366067d88c50f7e829138fde4f78d39b5b5802f1b92a8a820865af5cc79f9f30bc3f461c66af95d13e5e1f0381c184572a91dee1c849048a647a1158cf884064deddbf1b0b88dfe2f791428d0ba0f6fb2f04e14081f69165ae66d9297c118f0907705c9c4954a199bae0bb96fad763d690e7daa6cfda59ba7f2c8d11448b604d12d + +# References + +[sphinx]: http://www.cypherpunks.ca/~iang/pubs/Sphinx_Oakland09.pdf +[RFC2104]: https://tools.ietf.org/html/rfc2104 +[fips198]: http://csrc.nist.gov/publications/fips/fips198-1/FIPS-198-1_final.pdf +[sec2]: http://www.secg.org/sec2-v2.pdf +[rfc7539]: https://tools.ietf.org/html/rfc7539 + +# Authors + +[ FIXME: ] + +![Creative Commons License](https://i.creativecommons.org/l/by/4.0/88x31.png "License CC-BY") +
+This work is licensed under a [Creative Commons Attribution 4.0 International License](http://creativecommons.org/licenses/by/4.0/). +""" diff --git a/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py new file mode 100644 index 000000000..4f6bc8b19 --- /dev/null +++ b/contrib/pyln-spec/bolt4/pyln/spec/bolt4/gen_version.py @@ -0,0 +1,2 @@ +__version__ = "1.0.post137" +__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56" diff --git a/contrib/pyln-spec/bolt4/requirements.txt b/contrib/pyln-spec/bolt4/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/contrib/pyln-spec/bolt4/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/contrib/pyln-spec/bolt4/setup.py b/contrib/pyln-spec/bolt4/setup.py new file mode 100644 index 000000000..52f1d0b07 --- /dev/null +++ b/contrib/pyln-spec/bolt4/setup.py @@ -0,0 +1,23 @@ +from pyln.spec.bolt4 import __version__, desc +from setuptools import setup +import io + +with io.open('requirements.txt', encoding='utf-8') as f: + requirements = [r for r in f.read().split('\n') if len(r)] + + +def do_setup(boltnum: int, version: str, desc: str): + setup(name='pyln-bolt{}'.format(boltnum), + version=version, + description=desc, + url='http://github.com/ElementsProject/lightning', + author='Rusty Russell', + author_email='rusty@rustcorp.com.au', + license='MIT', + packages=['pyln.spec.bolt{}'.format(boltnum)], + scripts=[], + zip_safe=True, + install_requires=requirements) + + +do_setup(4, __version__, desc) diff --git a/contrib/pyln-proto/tests/test_bolt4.py b/contrib/pyln-spec/bolt4/tests/test_bolt4.py similarity index 78% rename from contrib/pyln-proto/tests/test_bolt4.py rename to contrib/pyln-spec/bolt4/tests/test_bolt4.py index 460611609..196f43e14 100644 --- a/contrib/pyln-proto/tests/test_bolt4.py +++ b/contrib/pyln-spec/bolt4/tests/test_bolt4.py @@ -1,6 +1,6 @@ #! /usr/bin/python3 from pyln.proto.message import MessageNamespace -import pyln.proto.message.bolt4 as bolt4 +import pyln.spec.bolt4 as bolt4 # FIXME: more tests diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/__init__.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/__init__.py new file mode 120000 index 000000000..52f300f8a --- /dev/null +++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/__init__.py @@ -0,0 +1 @@ +../../../../subinit.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/bolt.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/bolt.py new file mode 120000 index 000000000..c22b879fc --- /dev/null +++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/bolt.py @@ -0,0 +1 @@ +../../../../bolt.py \ No newline at end of file diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen.py new file mode 100644 index 000000000..bbd28edce --- /dev/null +++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen.py @@ -0,0 +1,1211 @@ +csv = [ + "msgtype,announcement_signatures,259", + "msgdata,announcement_signatures,channel_id,channel_id,", + "msgdata,announcement_signatures,short_channel_id,short_channel_id,", + "msgdata,announcement_signatures,node_signature,signature,", + "msgdata,announcement_signatures,bitcoin_signature,signature,", + "msgtype,channel_announcement,256", + "msgdata,channel_announcement,node_signature_1,signature,", + "msgdata,channel_announcement,node_signature_2,signature,", + "msgdata,channel_announcement,bitcoin_signature_1,signature,", + "msgdata,channel_announcement,bitcoin_signature_2,signature,", + "msgdata,channel_announcement,len,u16,", + "msgdata,channel_announcement,features,byte,len", + "msgdata,channel_announcement,chain_hash,chain_hash,", + "msgdata,channel_announcement,short_channel_id,short_channel_id,", + "msgdata,channel_announcement,node_id_1,point,", + "msgdata,channel_announcement,node_id_2,point,", + "msgdata,channel_announcement,bitcoin_key_1,point,", + "msgdata,channel_announcement,bitcoin_key_2,point,", + "msgtype,node_announcement,257", + "msgdata,node_announcement,signature,signature,", + "msgdata,node_announcement,flen,u16,", + "msgdata,node_announcement,features,byte,flen", + "msgdata,node_announcement,timestamp,u32,", + "msgdata,node_announcement,node_id,point,", + "msgdata,node_announcement,rgb_color,byte,3", + "msgdata,node_announcement,alias,byte,32", + "msgdata,node_announcement,addrlen,u16,", + "msgdata,node_announcement,addresses,byte,addrlen", + "msgtype,channel_update,258", + "msgdata,channel_update,signature,signature,", + "msgdata,channel_update,chain_hash,chain_hash,", + "msgdata,channel_update,short_channel_id,short_channel_id,", + "msgdata,channel_update,timestamp,u32,", + "msgdata,channel_update,message_flags,byte,", + "msgdata,channel_update,channel_flags,byte,", + "msgdata,channel_update,cltv_expiry_delta,u16,", + "msgdata,channel_update,htlc_minimum_msat,u64,", + "msgdata,channel_update,fee_base_msat,u32,", + "msgdata,channel_update,fee_proportional_millionths,u32,", + "msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max", + "msgtype,query_short_channel_ids,261,gossip_queries", + "msgdata,query_short_channel_ids,chain_hash,chain_hash,", + "msgdata,query_short_channel_ids,len,u16,", + "msgdata,query_short_channel_ids,encoded_short_ids,byte,len", + "msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,", + "tlvtype,query_short_channel_ids_tlvs,query_flags,1", + "tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,byte,", + "tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...", + "msgtype,reply_short_channel_ids_end,262,gossip_queries", + "msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,", + "msgdata,reply_short_channel_ids_end,full_information,byte,", + "msgtype,query_channel_range,263,gossip_queries", + "msgdata,query_channel_range,chain_hash,chain_hash,", + "msgdata,query_channel_range,first_blocknum,u32,", + "msgdata,query_channel_range,number_of_blocks,u32,", + "msgdata,query_channel_range,tlvs,query_channel_range_tlvs,", + "tlvtype,query_channel_range_tlvs,query_option,1", + "tlvdata,query_channel_range_tlvs,query_option,query_option_flags,bigsize,", + "msgtype,reply_channel_range,264,gossip_queries", + "msgdata,reply_channel_range,chain_hash,chain_hash,", + "msgdata,reply_channel_range,first_blocknum,u32,", + "msgdata,reply_channel_range,number_of_blocks,u32,", + "msgdata,reply_channel_range,full_information,byte,", + "msgdata,reply_channel_range,len,u16,", + "msgdata,reply_channel_range,encoded_short_ids,byte,len", + "msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,", + "tlvtype,reply_channel_range_tlvs,timestamps_tlv,1", + "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,byte,", + "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...", + "tlvtype,reply_channel_range_tlvs,checksums_tlv,3", + "tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...", + "subtype,channel_update_timestamps", + "subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,", + "subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,", + "subtype,channel_update_checksums", + "subtypedata,channel_update_checksums,checksum_node_id_1,u32,", + "subtypedata,channel_update_checksums,checksum_node_id_2,u32,", + "msgtype,gossip_timestamp_filter,265,gossip_queries", + "msgdata,gossip_timestamp_filter,chain_hash,chain_hash,", + "msgdata,gossip_timestamp_filter,first_timestamp,u32,", + "msgdata,gossip_timestamp_filter,timestamp_range,u32,", +] +desc = "BOLT #7: P2P Node and Channel Discovery" +text = """# BOLT #7: P2P Node and Channel Discovery + +This specification describes simple node discovery, channel discovery, and channel update mechanisms that do not rely on a third-party to disseminate the information. + +Node and channel discovery serve two different purposes: + + - Node discovery allows nodes to broadcast their ID, host, and port, so that other nodes can open connections and establish payment channels with them. + - Channel discovery allows the creation and maintenance of a local view of the network's topology, so that a node can discover routes to desired destinations. + +To support channel and node discovery, three *gossip messages* are supported: + +- For node discovery, peers exchange `node_announcement` + messages, which supply additional information about the nodes. There may be + multiple `node_announcement` messages, in order to update the node information. + + - For channel discovery, peers in the network exchange + `channel_announcement` messages containing information regarding new + channels between the two nodes. They can also exchange `channel_update` + messages, which update information about a channel. There can only be + one valid `channel_announcement` for any channel, but at least two + `channel_update` messages are expected. + +# Table of Contents + + * [Definition of `short_channel_id`](#definition-of-short_channel_id) + * [The `announcement_signatures` Message](#the-announcement_signatures-message) + * [The `channel_announcement` Message](#the-channel_announcement-message) + * [The `node_announcement` Message](#the-node_announcement-message) + * [The `channel_update` Message](#the-channel_update-message) + * [Query Messages](#query-messages) + * [Initial Sync](#initial-sync) + * [Rebroadcasting](#rebroadcasting) + * [HTLC Fees](#htlc-fees) + * [Pruning the Network View](#pruning-the-network-view) + * [Recommendations for Routing](#recommendations-for-routing) + * [References](#references) + +## Definition of `short_channel_id` + +The `short_channel_id` is the unique description of the funding transaction. +It is constructed as follows: + 1. the most significant 3 bytes: indicating the block height + 2. the next 3 bytes: indicating the transaction index within the block + 3. the least significant 2 bytes: indicating the output index that pays to the channel. + +The standard human readable format for `short_channel_id` is created +by printing the above components, in the order: +block height, transaction index, and output index. +Each component is printed as a decimal number, +and separated from each other by the small letter `x`. +For example, a `short_channel_id` might be written as `539268x845x1`, +indicating a channel on the output 1 of the transaction at index 845 +of the block at height 539268. + +### Rationale + +The `short_channel_id` human readable format is designed +so that double-clicking or double-tapping it will select the entire ID +on most systems. +Humans prefer decimal when reading numbers, +so the ID components are written in decimal. +The small letter `x` is used since on most fonts, +the `x` is visibly smaller than decimal digits, +making it easy to visibly group each component of the ID. + +## The `announcement_signatures` Message + +This is a direct message between the two endpoints of a channel and serves as an opt-in mechanism to allow the announcement of the channel to the rest of the network. +It contains the necessary signatures, by the sender, to construct the `channel_announcement` message. + +1. type: 259 (`announcement_signatures`) +2. data: + * [`channel_id`:`channel_id`] + * [`short_channel_id`:`short_channel_id`] + * [`signature`:`node_signature`] + * [`signature`:`bitcoin_signature`] + +The willingness of the initiating node to announce the channel is signaled during channel opening by setting the `announce_channel` bit in `channel_flags` (see [BOLT #2](02-peer-protocol.md#the-open_channel-message)). + +### Requirements + +The `announcement_signatures` message is created by constructing a `channel_announcement` message, corresponding to the newly established channel, and signing it with the secrets matching an endpoint's `node_id` and `bitcoin_key`. After it's signed, the +`announcement_signatures` message may be sent. + +A node: + - if the `open_channel` message has the `announce_channel` bit set AND a `shutdown` message has not been sent: + - MUST send the `announcement_signatures` message. + - MUST NOT send `announcement_signatures` messages until `funding_locked` + has been sent and received AND the funding transaction has at least six confirmations. + - otherwise: + - MUST NOT send the `announcement_signatures` message. + - upon reconnection (once the above timing requirements have been met): + - MUST respond to the first `announcement_signatures` message with its own + `announcement_signatures` message. + - if it has NOT received an `announcement_signatures` message: + - SHOULD retransmit the `announcement_signatures` message. + +A recipient node: + - if the `short_channel_id` is NOT correct: + - SHOULD fail the channel. + - if the `node_signature` OR the `bitcoin_signature` is NOT correct: + - MAY fail the channel. + - if it has sent AND received a valid `announcement_signatures` message: + - SHOULD queue the `channel_announcement` message for its peers. + - if it has not sent funding_locked: + - MAY defer handling the announcement_signatures until after it has sent funding_locked + - otherwise: + - MUST ignore it. + + +### Rationale + +The reason for allowing deferring of a premature announcement_signatures is +that an earlier version of the spec did not require waiting for receipt of +funding locked: deferring rather than ignoring it allows compatibility with +this behavior. + +## The `channel_announcement` Message + +This gossip message contains ownership information regarding a channel. It ties +each on-chain Bitcoin key to the associated Lightning node key, and vice-versa. +The channel is not practically usable until at least one side has announced +its fee levels and expiry, using `channel_update`. + +Proving the existence of a channel between `node_1` and `node_2` requires: + +1. proving that the funding transaction pays to `bitcoin_key_1` and + `bitcoin_key_2` +2. proving that `node_1` owns `bitcoin_key_1` +3. proving that `node_2` owns `bitcoin_key_2` + +Assuming that all nodes know the unspent transaction outputs, the first proof is +accomplished by a node finding the output given by the `short_channel_id` and +verifying that it is indeed a P2WSH funding transaction output for those keys +specified in [BOLT #3](03-transactions.md#funding-transaction-output). + +The last two proofs are accomplished through explicit signatures: +`bitcoin_signature_1` and `bitcoin_signature_2` are generated for each +`bitcoin_key` and each of the corresponding `node_id`s are signed. + +It's also necessary to prove that `node_1` and `node_2` both agree on the +announcement message: this is accomplished by having a signature from each +`node_id` (`node_signature_1` and `node_signature_2`) signing the message. + +1. type: 256 (`channel_announcement`) +2. data: + * [`signature`:`node_signature_1`] + * [`signature`:`node_signature_2`] + * [`signature`:`bitcoin_signature_1`] + * [`signature`:`bitcoin_signature_2`] + * [`u16`:`len`] + * [`len*byte`:`features`] + * [`chain_hash`:`chain_hash`] + * [`short_channel_id`:`short_channel_id`] + * [`point`:`node_id_1`] + * [`point`:`node_id_2`] + * [`point`:`bitcoin_key_1`] + * [`point`:`bitcoin_key_2`] + +### Requirements + +The origin node: + - MUST set `chain_hash` to the 32-byte hash that uniquely identifies the chain + that the channel was opened within: + - for the _Bitcoin blockchain_: + - MUST set `chain_hash` value (encoded in hex) equal to `6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000`. + - MUST set `short_channel_id` to refer to the confirmed funding transaction, + as specified in [BOLT #2](02-peer-protocol.md#the-funding_locked-message). + - Note: the corresponding output MUST be a P2WSH, as described in [BOLT #3](03-transactions.md#funding-transaction-output). + - MUST set `node_id_1` and `node_id_2` to the public keys of the two nodes + operating the channel, such that `node_id_1` is the lexicographically-lesser of the + two compressed keys sorted in ascending lexicographic order. + - MUST set `bitcoin_key_1` and `bitcoin_key_2` to `node_id_1` and `node_id_2`'s + respective `funding_pubkey`s. + - MUST compute the double-SHA256 hash `h` of the message, beginning at offset + 256, up to the end of the message. + - Note: the hash skips the 4 signatures but hashes the rest of the message, + including any future fields appended to the end. + - MUST set `node_signature_1` and `node_signature_2` to valid + signatures of the hash `h` (using `node_id_1` and `node_id_2`'s respective + secrets). + - MUST set `bitcoin_signature_1` and `bitcoin_signature_2` to valid + signatures of the hash `h` (using `bitcoin_key_1` and `bitcoin_key_2`'s + respective secrets). + - MUST set `features` based on what features were negotiated for this channel, according to [BOLT #9](09-features.md#assigned-features-flags) + - MUST set `len` to the minimum length required to hold the `features` bits + it sets. + +The receiving node: + - MUST verify the integrity AND authenticity of the message by verifying the + signatures. + - if there is an unknown even bit in the `features` field: + - MUST NOT attempt to route messages through the channel. + - if the `short_channel_id`'s output does NOT correspond to a P2WSH (using + `bitcoin_key_1` and `bitcoin_key_2`, as specified in + [BOLT #3](03-transactions.md#funding-transaction-output)) OR the output is + spent: + - MUST ignore the message. + - if the specified `chain_hash` is unknown to the receiver: + - MUST ignore the message. + - otherwise: + - if `bitcoin_signature_1`, `bitcoin_signature_2`, `node_signature_1` OR + `node_signature_2` are invalid OR NOT correct: + - SHOULD fail the connection. + - otherwise: + - if `node_id_1` OR `node_id_2` are blacklisted: + - SHOULD ignore the message. + - otherwise: + - if the transaction referred to was NOT previously announced as a + channel: + - SHOULD queue the message for rebroadcasting. + - MAY choose NOT to for messages longer than the minimum expected + length. + - if it has previously received a valid `channel_announcement`, for the + same transaction, in the same block, but for a different `node_id_1` or + `node_id_2`: + - SHOULD blacklist the previous message's `node_id_1` and `node_id_2`, + as well as this `node_id_1` and `node_id_2` AND forget any channels + connected to them. + - otherwise: + - SHOULD store this `channel_announcement`. + - once its funding output has been spent OR reorganized out: + - SHOULD forget a channel. + +### Rationale + +Both nodes are required to sign to indicate they are willing to route other +payments via this channel (i.e. be part of the public network); requiring their +Bitcoin signatures proves that they control the channel. + +The blacklisting of conflicting nodes disallows multiple different +announcements. Such conflicting announcements should never be broadcast by any +node, as this implies that keys have leaked. + +While channels should not be advertised before they are sufficiently deep, the +requirement against rebroadcasting only applies if the transaction has not moved +to a different block. + +In order to avoid storing excessively large messages, yet still allow for +reasonable future expansion, nodes are permitted to restrict rebroadcasting +(perhaps statistically). + +New channel features are possible in the future: backwards compatible (or +optional) features will have _odd_ feature bits, while incompatible features +will have _even_ feature bits +(["It's OK to be odd!"](00-introduction.md#glossary-and-terminology-guide)). + +## The `node_announcement` Message + +This gossip message allows a node to indicate extra data associated with it, in +addition to its public key. To avoid trivial denial of service attacks, +nodes not associated with an already known channel are ignored. + +1. type: 257 (`node_announcement`) +2. data: + * [`signature`:`signature`] + * [`u16`:`flen`] + * [`flen*byte`:`features`] + * [`u32`:`timestamp`] + * [`point`:`node_id`] + * [`3*byte`:`rgb_color`] + * [`32*byte`:`alias`] + * [`u16`:`addrlen`] + * [`addrlen*byte`:`addresses`] + +`timestamp` allows for the ordering of messages, in the case of multiple +announcements. `rgb_color` and `alias` allow intelligence services to assign +nodes colors like black and cool monikers like 'IRATEMONK' and 'WISTFULTOLL'. + +`addresses` allows a node to announce its willingness to accept incoming network +connections: it contains a series of `address descriptor`s for connecting to the +node. The first byte describes the address type and is followed by the +appropriate number of bytes for that type. + +The following `address descriptor` types are defined: + + * `1`: ipv4; data = `[4:ipv4_addr][2:port]` (length 6) + * `2`: ipv6; data = `[16:ipv6_addr][2:port]` (length 18) + * `3`: Tor v2 onion service; data = `[10:onion_addr][2:port]` (length 12) + * version 2 onion service addresses; Encodes an 80-bit, truncated `SHA-1` + hash of a 1024-bit `RSA` public key for the onion service (a.k.a. Tor + hidden service). + * `4`: Tor v3 onion service; data = `[35:onion_addr][2:port]` (length 37) + * version 3 ([prop224](https://gitweb.torproject.org/torspec.git/tree/proposals/224-rend-spec-ng.txt)) + onion service addresses; Encodes: + `[32:32_byte_ed25519_pubkey] || [2:checksum] || [1:version]`, where + `checksum = sha3(".onion checksum" | pubkey || version)[:2]`. + +### Requirements + +The origin node: + - MUST set `timestamp` to be greater than that of any previous + `node_announcement` it has previously created. + - MAY base it on a UNIX timestamp. + - MUST set `signature` to the signature of the double-SHA256 of the entire + remaining packet after `signature` (using the key given by `node_id`). + - MAY set `alias` AND `rgb_color` to customize its appearance in maps and + graphs. + - Note: the first byte of `rgb_color` is the red value, the second byte is the + green value, and the last byte is the blue value. + - MUST set `alias` to a valid UTF-8 string, with any `alias` trailing-bytes + equal to 0. + - SHOULD fill `addresses` with an address descriptor for each public network + address that expects incoming connections. + - MUST set `addrlen` to the number of bytes in `addresses`. + - MUST place address descriptors in ascending order. + - SHOULD NOT place any zero-typed address descriptors anywhere. + - SHOULD use placement only for aligning fields that follow `addresses`. + - MUST NOT create a `type 1` OR `type 2` address descriptor with `port` equal + to 0. + - SHOULD ensure `ipv4_addr` AND `ipv6_addr` are routable addresses. + - MUST set `features` according to [BOLT #9](09-features.md#assigned-features-flags) + - SHOULD set `flen` to the minimum length required to hold the `features` + bits it sets. + +The receiving node: + - if `node_id` is NOT a valid compressed public key: + - SHOULD fail the connection. + - MUST NOT process the message further. + - if `signature` is NOT a valid signature (using `node_id` of the + double-SHA256 of the entire message following the `signature` field, including +any future fields appended to the end): + - SHOULD fail the connection. + - MUST NOT process the message further. + - if `features` field contains _unknown even bits_: + - SHOULD NOT connect to the node. + - Unless paying a [BOLT #11](11-payment-encoding.md) invoice which does not + have the same bit(s) set, MUST NOT attempt to send payments _to_ the node. + - MUST NOT route a payment _through_ the node. + - SHOULD ignore the first `address descriptor` that does NOT match the types + defined above. + - if `addrlen` is insufficient to hold the address descriptors of the + known types: + - SHOULD fail the connection. + - if `port` is equal to 0: + - SHOULD ignore `ipv6_addr` OR `ipv4_addr`. + - if `node_id` is NOT previously known from a `channel_announcement` message, + OR if `timestamp` is NOT greater than the last-received `node_announcement` + from this `node_id`: + - SHOULD ignore the message. + - otherwise: + - if `timestamp` is greater than the last-received `node_announcement` from + this `node_id`: + - SHOULD queue the message for rebroadcasting. + - MAY choose NOT to queue messages longer than the minimum expected length. + - MAY use `rgb_color` AND `alias` to reference nodes in interfaces. + - SHOULD insinuate their self-signed origins. + +### Rationale + +New node features are possible in the future: backwards compatible (or +optional) ones will have _odd_ `feature` _bits_, incompatible ones will have +_even_ `feature` _bits_. These will be propagated normally; incompatible +feature bits here refer to the nodes, not the `node_announcement` message +itself. + +New address types may be added in the future; as address descriptors have +to be ordered in ascending order, unknown ones can be safely ignored. +Additional fields beyond `addresses` may also be added in the future—with +optional padding within `addresses`, if they require certain alignment. + +### Security Considerations for Node Aliases + +Node aliases are user-defined and provide a potential avenue for injection +attacks, both during the process of rendering and during persistence. + +Node aliases should always be sanitized before being displayed in +HTML/Javascript contexts or any other dynamically interpreted rendering +frameworks. Similarly, consider using prepared statements, input validation, +and escaping to protect against injection vulnerabilities and persistence +engines that support SQL or other dynamically interpreted querying languages. + +* [Stored and Reflected XSS Prevention](https://www.owasp.org/index.php/XSS_(Cross_Site_Scripting)_Prevention_Cheat_Sheet) +* [DOM-based XSS Prevention](https://www.owasp.org/index.php/DOM_based_XSS_Prevention_Cheat_Sheet) +* [SQL Injection Prevention](https://www.owasp.org/index.php/SQL_Injection_Prevention_Cheat_Sheet) + +Don't be like the school of [Little Bobby Tables](https://xkcd.com/327/). + +## The `channel_update` Message + +After a channel has been initially announced, each side independently +announces the fees and minimum expiry delta it requires to relay HTLCs +through this channel. Each uses the 8-byte channel shortid that matches the +`channel_announcement` and the 1-bit `channel_flags` field to indicate which end of the +channel it's on (origin or final). A node can do this multiple times, in +order to change fees. + +Note that the `channel_update` gossip message is only useful in the context +of *relaying* payments, not *sending* payments. When making a payment + `A` -> `B` -> `C` -> `D`, only the `channel_update`s related to channels + `B` -> `C` (announced by `B`) and `C` -> `D` (announced by `C`) will + come into play. When building the route, amounts and expiries for HTLCs need + to be calculated backward from the destination to the source. The exact initial + value for `amount_msat` and the minimal value for `cltv_expiry`, to be used for + the last HTLC in the route, are provided in the payment request + (see [BOLT #11](11-payment-encoding.md#tagged-fields)). + +1. type: 258 (`channel_update`) +2. data: + * [`signature`:`signature`] + * [`chain_hash`:`chain_hash`] + * [`short_channel_id`:`short_channel_id`] + * [`u32`:`timestamp`] + * [`byte`:`message_flags`] + * [`byte`:`channel_flags`] + * [`u16`:`cltv_expiry_delta`] + * [`u64`:`htlc_minimum_msat`] + * [`u32`:`fee_base_msat`] + * [`u32`:`fee_proportional_millionths`] + * [`u64`:`htlc_maximum_msat`] (option_channel_htlc_max) + +The `channel_flags` bitfield is used to indicate the direction of the channel: it +identifies the node that this update originated from and signals various options +concerning the channel. The following table specifies the meaning of its +individual bits: + +| Bit Position | Name | Meaning | +| ------------- | ----------- | -------------------------------- | +| 0 | `direction` | Direction this update refers to. | +| 1 | `disable` | Disable the channel. | + +The `message_flags` bitfield is used to indicate the presence of optional +fields in the `channel_update` message: + +| Bit Position | Name | Field | +| ------------- | ------------------------- | -------------------------------- | +| 0 | `option_channel_htlc_max` | `htlc_maximum_msat` | + +Note that the `htlc_maximum_msat` field is static in the current +protocol over the life of the channel: it is *not* designed to be +indicative of real-time channel capacity in each direction, which +would be both a massive data leak and uselessly spam the network (it +takes an average of 30 seconds for gossip to propagate each hop). + +The `node_id` for the signature verification is taken from the corresponding +`channel_announcement`: `node_id_1` if the least-significant bit of flags is 0 +or `node_id_2` otherwise. + +### Requirements + +The origin node: + - MUST NOT send a created `channel_update` before `funding_locked` has been received. + - MAY create a `channel_update` to communicate the channel parameters to the + channel peer, even though the channel has not yet been announced (i.e. the + `announce_channel` bit was not set). + - MUST NOT forward such a `channel_update` to other peers, for privacy + reasons. + - Note: such a `channel_update`, one not preceded by a + `channel_announcement`, is invalid to any other peer and would be discarded. + - MUST set `signature` to the signature of the double-SHA256 of the entire + remaining packet after `signature`, using its own `node_id`. + - MUST set `chain_hash` AND `short_channel_id` to match the 32-byte hash AND + 8-byte channel ID that uniquely identifies the channel specified in the + `channel_announcement` message. + - if the origin node is `node_id_1` in the message: + - MUST set the `direction` bit of `channel_flags` to 0. + - otherwise: + - MUST set the `direction` bit of `channel_flags` to 1. + - if the `htlc_maximum_msat` field is present: + - MUST set the `option_channel_htlc_max` bit of `message_flags` to 1. + - MUST set `htlc_maximum_msat` to the maximum value it will send through this channel for a single HTLC. + - MUST set this to less than or equal to the channel capacity. + - MUST set this to less than or equal to `max_htlc_value_in_flight_msat` + it received from the peer. + - for channels with `chain_hash` identifying the Bitcoin blockchain: + - MUST set this to less than 2^32. + - otherwise: + - MUST set the `option_channel_htlc_max` bit of `message_flags` to 0. + - MUST set bits in `channel_flags` and `message_flags `that are not assigned a meaning to 0. + - MAY create and send a `channel_update` with the `disable` bit set to 1, to + signal a channel's temporary unavailability (e.g. due to a loss of + connectivity) OR permanent unavailability (e.g. prior to an on-chain + settlement). + - MAY sent a subsequent `channel_update` with the `disable` bit set to 0 to + re-enable the channel. + - MUST set `timestamp` to greater than 0, AND to greater than any + previously-sent `channel_update` for this `short_channel_id`. + - SHOULD base `timestamp` on a UNIX timestamp. + - MUST set `cltv_expiry_delta` to the number of blocks it will subtract from + an incoming HTLC's `cltv_expiry`. + - MUST set `htlc_minimum_msat` to the minimum HTLC value (in millisatoshi) + that the channel peer will accept. + - MUST set `fee_base_msat` to the base fee (in millisatoshi) it will charge + for any HTLC. + - MUST set `fee_proportional_millionths` to the amount (in millionths of a + satoshi) it will charge per transferred satoshi. + - SHOULD NOT create redundant `channel_update`s + +The receiving node: + - if the `short_channel_id` does NOT match a previous `channel_announcement`, + OR if the channel has been closed in the meantime: + - MUST ignore `channel_update`s that do NOT correspond to one of its own + channels. + - SHOULD accept `channel_update`s for its own channels (even if non-public), + in order to learn the associated origin nodes' forwarding parameters. + - if `signature` is not a valid signature, using `node_id` of the + double-SHA256 of the entire message following the `signature` field (including + unknown fields following `fee_proportional_millionths`): + - MUST NOT process the message further. + - SHOULD fail the connection. + - if the specified `chain_hash` value is unknown (meaning it isn't active on + the specified chain): + - MUST ignore the channel update. + - if the `timestamp` is equal to the last-received `channel_update` for this + `short_channel_id` AND `node_id`: + - if the fields below `timestamp` differ: + - MAY blacklist this `node_id`. + - MAY forget all channels associated with it. + - if the fields below `timestamp` are equal: + - SHOULD ignore this message + - if `timestamp` is lower than that of the last-received + `channel_update` for this `short_channel_id` AND for `node_id`: + - SHOULD ignore the message. + - otherwise: + - if the `timestamp` is unreasonably far in the future: + - MAY discard the `channel_update`. + - otherwise: + - SHOULD queue the message for rebroadcasting. + - MAY choose NOT to for messages longer than the minimum expected length. + - if the `option_channel_htlc_max` bit of `message_flags` is 0: + - MUST consider `htlc_maximum_msat` not to be present. + - otherwise: + - if `htlc_maximum_msat` is not present or greater than channel capacity: + - MAY blacklist this `node_id` + - SHOULD ignore this channel during route considerations. + - otherwise: + - SHOULD consider the `htlc_maximum_msat` when routing. + +### Rationale + +The `timestamp` field is used by nodes for pruning `channel_update`s that are +either too far in the future or have not been updated in two weeks; so it +makes sense to have it be a UNIX timestamp (i.e. seconds since UTC +1970-01-01). This cannot be a hard requirement, however, given the possible case +of two `channel_update`s within a single second. + +It is assumed that more than one `channel_update` message changing the channel +parameters in the same second may be a DoS attempt, and therefore, the node responsible +for signing such messages may be blacklisted. However, a node may send a same +`channel_update` message with a different signature (changing the nonce in signature +signing), and hence fields apart from signature are checked to see if the channel +parameters have changed for the same timestamp. It is also important to note that +ECDSA signatures are malleable. So, an intermediate node who received the `channel_update` +message can rebroadcast it just by changing the `s` component of signature with `-s`. +This should however not result in the blacklist of the `node_id` from where +the message originated. + +The explicit `option_channel_htlc_max` flag to indicate the presence +of `htlc_maximum_msat` (rather than having `htlc_maximum_msat` implied +by the message length) allows us to extend the `channel_update` +with different fields in future. Since channels are limited to 2^32-1 +millisatoshis in Bitcoin, the `htlc_maximum_msat` has the same restriction. + +The recommendation against redundant `channel_update`s minimizes spamming the network, +however it is sometimes inevitable. For example, a channel with a +peer which is unreachable will eventually cause a `channel_update` to +indicate that the channel is disabled, with another update re-enabling +the channel when the peer reestablishes contact. Because gossip +messages are batched and replace previous ones, the result may be a +single seemingly-redundant update. + +## Query Messages + +Negotiating the `gossip_queries` option via `init` enables a number +of extended queries for gossip synchronization. These explicitly +request what gossip should be received. + +There are several messages which contain a long array of +`short_channel_id`s (called `encoded_short_ids`) so we utilize a +simple compression scheme: the first byte indicates the encoding, the +rest contains the data. + +Encoding types: +* `0`: uncompressed array of `short_channel_id` types, in ascending order. +* `1`: array of `short_channel_id` types, in ascending order, compressed with zlib deflate[1](#reference-1) + +This encoding is also used for arrays of other types (timestamps, flags, ...), and specified with an `encoded_` prefix. For example, `encoded_timestamps` is an array of timestamps than can be either compressed (with a `1` prefix) or uncompressed (with a `0` prefix). + +Note that a 65535-byte zlib message can decompress into 67632120 +bytes[2](#reference-2), but since the only valid contents +are unique 8-byte values, no more than 14 bytes can be duplicated +across the stream: as each duplicate takes at least 2 bits, no valid +contents could decompress to more than 3669960 bytes. + +Query messages can be extended with optional fields that can help reduce the number of messages needed to synchronize routing tables by enabling: + +- timestamp-based filtering of `channel_update` messages: only ask for `channel_update` messages that are newer than the ones you already have. +- checksum-based filtering of `channel_update` messages: only ask for `channel_update` messages that carry different information from the ones you already have. + +Nodes can signal that they support extended gossip queries with the `gossip_queries_ex` feature bit. + +### The `query_short_channel_ids`/`reply_short_channel_ids_end` Messages + +1. type: 261 (`query_short_channel_ids`) (`gossip_queries`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`u16`:`len`] + * [`len*byte`:`encoded_short_ids`] + * [`query_short_channel_ids_tlvs`:`tlvs`] + +1. tlvs: `query_short_channel_ids_tlvs` +2. types: + 1. type: 1 (`query_flags`) + 2. data: + * [`byte`:`encoding_type`] + * [`...*byte`:`encoded_query_flags`] + +`encoded_query_flags` is an array of bitfields, one bigsize per bitfield, one bitfield for each `short_channel_id`. Bits have the following meaning: + +| Bit Position | Meaning | +| ------------- | ---------------------------------------- | +| 0 | Sender wants `channel_announcement` | +| 1 | Sender wants `channel_update` for node 1 | +| 2 | Sender wants `channel_update` for node 2 | +| 3 | Sender wants `node_announcement` for node 1 | +| 4 | Sender wants `node_announcement` for node 2 | + +Query flags must be minimally encoded, which means that one flag will be encoded with a single byte. + +1. type: 262 (`reply_short_channel_ids_end`) (`gossip_queries`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`byte`:`full_information`] + +This is a general mechanism which lets a node query for the +`channel_announcement` and `channel_update` messages for specific channels +(identified via `short_channel_id`s). This is usually used either because +a node sees a `channel_update` for which it has no `channel_announcement` or +because it has obtained previously unknown `short_channel_id`s +from `reply_channel_range`. + +#### Requirements + +The sender: + - MUST NOT send `query_short_channel_ids` if it has sent a previous `query_short_channel_ids` to this peer and not received `reply_short_channel_ids_end`. + - MUST set `chain_hash` to the 32-byte hash that uniquely identifies the chain + that the `short_channel_id`s refer to. + - MUST set the first byte of `encoded_short_ids` to the encoding type. + - MUST encode a whole number of `short_channel_id`s to `encoded_short_ids` + - MAY send this if it receives a `channel_update` for a + `short_channel_id` for which it has no `channel_announcement`. + - SHOULD NOT send this if the channel referred to is not an unspent output. + - MAY include an optional `query_flags`. If so: + - MUST set `encoding_type`, as for `encoded_short_ids`. + - Each query flag is a minimally-encoded bigsize. + - MUST encode one query flag per `short_channel_id`. + +The receiver: + - if the first byte of `encoded_short_ids` is not a known encoding type: + - MAY fail the connection + - if `encoded_short_ids` does not decode into a whole number of `short_channel_id`: + - MAY fail the connection. + - if it has not sent `reply_short_channel_ids_end` to a previously received `query_short_channel_ids` from this sender: + - MAY fail the connection. + - if the incoming message includes `query_short_channel_ids_tlvs`: + - if `encoding_type` is not a known encoding type: + - MAY fail the connection + - if `encoded_query_flags` does not decode to exactly one flag per `short_channel_id`: + - MAY fail the connection. + - MUST respond to each known `short_channel_id`: + - if the incoming message does not include `encoded_query_flags`: + - with a `channel_announcement` and the latest `channel_update` for each end + - MUST follow with any `node_announcement`s for each `channel_announcement` + - otherwise: + - We define `query_flag` for the Nth `short_channel_id` in + `encoded_short_ids` to be the Nth bigsize of the decoded + `encoded_query_flags`. + - if bit 0 of `query_flag` is set: + - MUST reply with a `channel_announcement` + - if bit 1 of `query_flag` is set and it has received a `channel_update` from `node_id_1`: + - MUST reply with the latest `channel_update` for `node_id_1` + - if bit 2 of `query_flag` is set and it has received a `channel_update` from `node_id_2`: + - MUST reply with the latest `channel_update` for `node_id_2` + - if bit 3 of `query_flag` is set and it has received a `node_announcement` from `node_id_1`: + - MUST reply with the latest `node_announcement` for `node_id_1` + - if bit 4 of `query_flag` is set and it has received a `node_announcement` from `node_id_2`: + - MUST reply with the latest `node_announcement` for `node_id_2` + - SHOULD NOT wait for the next outgoing gossip flush to send these. + - SHOULD avoid sending duplicate `node_announcements` in response to a single `query_short_channel_ids`. + - MUST follow these responses with `reply_short_channel_ids_end`. + - if does not maintain up-to-date channel information for `chain_hash`: + - MUST set `full_information` to 0. + - otherwise: + - SHOULD set `full_information` to 1. + +#### Rationale + +Future nodes may not have complete information; they certainly won't have +complete information on unknown `chain_hash` chains. While this `full_information` +field (previously and confusingly called `complete`) cannot be trusted, a 0 does indicate that the sender should search +elsewhere for additional data. + +The explicit `reply_short_channel_ids_end` message means that the receiver can +indicate it doesn't know anything, and the sender doesn't need to rely on +timeouts. It also causes a natural ratelimiting of queries. + +### The `query_channel_range` and `reply_channel_range` Messages + +1. type: 263 (`query_channel_range`) (`gossip_queries`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`u32`:`first_blocknum`] + * [`u32`:`number_of_blocks`] + * [`query_channel_range_tlvs`:`tlvs`] + +1. tlvs: `query_channel_range_tlvs` +2. types: + 1. type: 1 (`query_option`) + 2. data: + * [`bigsize`:`query_option_flags`] + +`query_option_flags` is a bitfield represented as a minimally-encoded bigsize. Bits have the following meaning: + +| Bit Position | Meaning | +| ------------- | ----------------------- | +| 0 | Sender wants timestamps | +| 1 | Sender wants checksums | + +Though it is possible, it would not be very useful to ask for checksums without asking for timestamps too: the receiving node may have an older `channel_update` with a different checksum, asking for it would be useless. And if a `channel_update` checksum is actually 0 (which is quite unlikely) it will not be queried. + +1. type: 264 (`reply_channel_range`) (`gossip_queries`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`u32`:`first_blocknum`] + * [`u32`:`number_of_blocks`] + * [`byte`:`full_information`] + * [`u16`:`len`] + * [`len*byte`:`encoded_short_ids`] + * [`reply_channel_range_tlvs`:`tlvs`] + +1. tlvs: `reply_channel_range_tlvs` +2. types: + 1. type: 1 (`timestamps_tlv`) + 2. data: + * [`byte`:`encoding_type`] + * [`...*byte`:`encoded_timestamps`] + 1. type: 3 (`checksums_tlv`) + 2. data: + * [`...*channel_update_checksums`:`checksums`] + +For a single `channel_update`, timestamps are encoded as: + +1. subtype: `channel_update_timestamps` +2. data: + * [`u32`:`timestamp_node_id_1`] + * [`u32`:`timestamp_node_id_2`] + +Where: +* `timestamp_node_id_1` is the timestamp of the `channel_update` for `node_id_1`, or 0 if there was no `channel_update` from that node. +* `timestamp_node_id_2` is the timestamp of the `channel_update` for `node_id_2`, or 0 if there was no `channel_update` from that node. + +For a single `channel_update`, checksums are encoded as: + +1. subtype: `channel_update_checksums` +2. data: + * [`u32`:`checksum_node_id_1`] + * [`u32`:`checksum_node_id_2`] + +Where: +* `checksum_node_id_1` is the checksum of the `channel_update` for `node_id_1`, or 0 if there was no `channel_update` from that node. +* `checksum_node_id_2` is the checksum of the `channel_update` for `node_id_2`, or 0 if there was no `channel_update` from that node. + +The checksum of a `channel_update` is the CRC32C checksum as specified in [RFC3720](https://tools.ietf.org/html/rfc3720#appendix-B.4) of this `channel_update` without its `signature` and `timestamp` fields. + +This allows to query for channels within specific blocks. + +#### Requirements + +The sender of `query_channel_range`: + - MUST NOT send this if it has sent a previous `query_channel_range` to this peer and not received all `reply_channel_range` replies. + - MUST set `chain_hash` to the 32-byte hash that uniquely identifies the chain + that it wants the `reply_channel_range` to refer to + - MUST set `first_blocknum` to the first block it wants to know channels for + - MUST set `number_of_blocks` to 1 or greater. + - MAY append an additional `query_channel_range_tlv`, which specifies the type of extended information it would like to receive. + +The receiver of `query_channel_range`: + - if it has not sent all `reply_channel_range` to a previously received `query_channel_range` from this sender: + - MAY fail the connection. + - MUST respond with one or more `reply_channel_range`: + - MUST set with `chain_hash` equal to that of `query_channel_range`, + - MUST limit `number_of_blocks` to the maximum number of blocks whose + results could fit in `encoded_short_ids` + - if does not maintain up-to-date channel information for `chain_hash`: + - MUST set `full_information` to 0. + - otherwise: + - SHOULD set `full_information` to 1. + - the first `reply_channel_range` message: + - MUST set `first_blocknum` less than or equal to the `first_blocknum` in `query_channel_range` + - MUST set `first_blocknum` plus `number_of_blocks` greater than `first_blocknum` in `query_channel_range`. + - successive `reply_channel_range` message: + - MUST set `first_blocknum` to the previous `first_blocknum` plus `number_of_blocks`. + - the final `reply_channel_range` message: + - MUST have `first_blocknum` plus `number_of_blocks` equal or greater than the `query_channel_range` `first_blocknum` plus `number_of_blocks`. + +If the incoming message includes `query_option`, the receiver MAY append additional information to its reply: +- if bit 0 in `query_option_flags` is set, the receiver MAY append a `timestamps_tlv` that contains `channel_update` timestamps for all `short_chanel_id`s in `encoded_short_ids` +- if bit 1 in `query_option_flags` is set, the receiver MAY append a `checksums_tlv` that contains `channel_update` checksums for all `short_chanel_id`s in `encoded_short_ids` + + +#### Rationale + +A single response might be too large for a single packet, so multiple replies +may be required. We want to allow a peer to store canned results for (say) +1000-block ranges, so replies can exceed the requested range. However, we +require that each reply be relevant (overlapping the requested range). + +By insisting that replies be in increasing order, the receiver can easily +determine if replies are done: simply check if `first_blocknum` plus +`number_of_blocks` equals or exceeds the `first_blocknum` plus +`number_of_blocks` it asked for. + +The addition of timestamp and checksum fields allow a peer to omit querying for redundant updates. + +### The `gossip_timestamp_filter` Message + +1. type: 265 (`gossip_timestamp_filter`) (`gossip_queries`) +2. data: + * [`chain_hash`:`chain_hash`] + * [`u32`:`first_timestamp`] + * [`u32`:`timestamp_range`] + +This message allows a node to constrain future gossip messages to +a specific range. A node which wants any gossip messages would have +to send this, otherwise `gossip_queries` negotiation means no gossip +messages would be received. + +Note that this filter replaces any previous one, so it can be used +multiple times to change the gossip from a peer. + +#### Requirements + +The sender: + - MUST set `chain_hash` to the 32-byte hash that uniquely identifies the chain + that it wants the gossip to refer to. + +The receiver: + - SHOULD send all gossip messages whose `timestamp` is greater or + equal to `first_timestamp`, and less than `first_timestamp` plus + `timestamp_range`. + - MAY wait for the next outgoing gossip flush to send these. + - SHOULD send gossip messages as it generates them regardless of `timestamp`. + - Otherwise (relayed gossip): + - SHOULD restrict future gossip messages to those whose `timestamp` + is greater or equal to `first_timestamp`, and less than + `first_timestamp` plus `timestamp_range`. + - If a `channel_announcement` has no corresponding `channel_update`s: + - MUST NOT send the `channel_announcement`. + - Otherwise: + - MUST consider the `timestamp` of the `channel_announcement` to be the `timestamp` of a corresponding `channel_update`. + - MUST consider whether to send the `channel_announcement` after receiving the first corresponding `channel_update`. + - If a `channel_announcement` is sent: + - MUST send the `channel_announcement` prior to any corresponding `channel_update`s and `node_announcement`s. + +#### Rationale + +Since `channel_announcement` doesn't have a timestamp, we generate a likely +one. If there's no `channel_update` then it is not sent at all, which is most +likely in the case of pruned channels. + +Otherwise the `channel_announcement` is usually followed immediately by a +`channel_update`. Ideally we would specify that the first (oldest) `channel_update`'s +timestamp is to be used as the time of the `channel_announcement`, but new nodes on +the network will not have this, and further would require the first `channel_update` +timestamp to be stored. Instead, we allow any update to be used, which +is simple to implement. + +In the case where the `channel_announcement` is nonetheless missed, +`query_short_channel_ids` can be used to retrieve it. + +Nodes can use `timestamp_filter` to reduce their gossip load when they +have many peers (eg. setting `first_timestamp` to `0xFFFFFFFF` after the +first few peers, in the assumption that propagation is adequate). +This assumption of adequate propagation does not apply for gossip messages +generated directly by the node itself, so they should ignore filters. + +## Initial Sync + +If a node requires an initial sync of gossip messages, it will be flagged +in the `init` message, via a feature flag ([BOLT #9](09-features.md#assigned-localfeatures-flags)). + +Note that the `initial_routing_sync` feature is overridden (and should +be considered equal to 0) by the `gossip_queries` feature if the +latter is negotiated via `init`. + +Note that `gossip_queries` does not work with older nodes, so the +value of `initial_routing_sync` is still important to control +interactions with them. + +### Requirements + +A node: + - if the `gossip_queries` feature is negotiated: + - MUST NOT relay any gossip messages it did not generate itself, unless explicitly requested. + - otherwise: + - if it requires a full copy of the peer's routing state: + - SHOULD set the `initial_routing_sync` flag to 1. + - upon receiving an `init` message with the `initial_routing_sync` flag set to + 1: + - SHOULD send gossip messages for all known channels and nodes, as if they were just + received. + - if the `initial_routing_sync` flag is set to 0, OR if the initial sync was + completed: + - SHOULD resume normal operation, as specified in the following + [Rebroadcasting](#rebroadcasting) section. + +## Rebroadcasting + +### Requirements + +A receiving node: + - upon receiving a new `channel_announcement` or a `channel_update` or + `node_announcement` with an updated `timestamp`: + - SHOULD update its local view of the network's topology accordingly. + - after applying the changes from the announcement: + - if there are no channels associated with the corresponding origin node: + - MAY purge the origin node from its set of known nodes. + - otherwise: + - SHOULD update the appropriate metadata AND store the signature + associated with the announcement. + - Note: this will later allow the node to rebuild the announcement + for its peers. + +A node: + - if the `gossip_queries` feature is negotiated: + - MUST not send gossip it did not generate itself, until it receives `gossip_timestamp_filter`. + - SHOULD flush outgoing gossip messages once every 60 seconds, independently of + the arrival times of the messages. + - Note: this results in staggered announcements that are unique (not + duplicated). + - SHOULD NOT forward gossip messages to peers who sent `networks` in `init` + and did not specify the `chain_hash` of this gossip message. + - MAY re-announce its channels regularly. + - Note: this is discouraged, in order to keep the resource requirements low. + - upon connection establishment: + - SHOULD send all `channel_announcement` messages, followed by the latest + `node_announcement` AND `channel_update` messages. + +### Rationale + +Once the gossip message has been processed, it's added to a list of outgoing +messages, destined for the processing node's peers, replacing any older +updates from the origin node. This list of gossip messages will be flushed at +regular intervals; such a store-and-delayed-forward broadcast is called a +_staggered broadcast_. Also, such batching forms a natural rate +limit with low overhead. + +The sending of all gossip on reconnection is naive, but simple, +and allows bootstrapping for new nodes as well as updating for nodes that +have been offline for some time. The `gossip_queries` option +allows for more refined synchronization. + +## HTLC Fees + +### Requirements + +The origin node: + - SHOULD accept HTLCs that pay a fee equal to or greater than: + - fee_base_msat + ( amount_to_forward * fee_proportional_millionths / 1000000 ) + - SHOULD accept HTLCs that pay an older fee, for some reasonable time after + sending `channel_update`. + - Note: this allows for any propagation delay. + +## Pruning the Network View + +### Requirements + +A node: + - SHOULD monitor the funding transactions in the blockchain, to identify + channels that are being closed. + - if the funding output of a channel is being spent: + - SHOULD be removed from the local network view AND be considered closed. + - if the announced node no longer has any associated open channels: + - MAY prune nodes added through `node_announcement` messages from their + local view. + - Note: this is a direct result of the dependency of a `node_announcement` + being preceded by a `channel_announcement`. + +### Recommendation on Pruning Stale Entries + +#### Requirements + +A node: + - if a channel's latest `channel_update`s `timestamp` is older than two weeks + (1209600 seconds): + - MAY prune the channel. + - MAY ignore the channel. + - Note: this is an individual node policy and MUST NOT be enforced by + forwarding peers, e.g. by closing channels when receiving outdated gossip + messages. + +#### Rationale + +Several scenarios may result in channels becoming unusable and its endpoints +becoming unable to send updates for these channels. For example, this occurs if +both endpoints lose access to their private keys and can neither sign +`channel_update`s nor close the channel on-chain. In this case, the channels are +unlikely to be part of a computed route, since they would be partitioned off +from the rest of the network; however, they would remain in the local network +view would be forwarded to other peers indefinitely. + +## Recommendations for Routing + +When calculating a route for an HTLC, both the `cltv_expiry_delta` and the fee +need to be considered: the `cltv_expiry_delta` contributes to the time that +funds will be unavailable in the event of a worst-case failure. The relationship +between these two attributes is unclear, as it depends on the reliability of the +nodes involved. + +If a route is computed by simply routing to the intended recipient and summing +the `cltv_expiry_delta`s, then it's possible for intermediate nodes to guess +their position in the route. Knowing the CLTV of the HTLC, the surrounding +network topology, and the `cltv_expiry_delta`s gives an attacker a way to guess +the intended recipient. Therefore, it's highly desirable to add a random offset +to the CLTV that the intended recipient will receive, which bumps all CLTVs +along the route. + +In order to create a plausible offset, the origin node MAY start a limited +random walk on the graph, starting from the intended recipient and summing the +`cltv_expiry_delta`s, and use the resulting sum as the offset. +This effectively creates a _shadow route extension_ to the actual route and +provides better protection against this attack vector than simply picking a +random offset would. + +Other more advanced considerations involve diversification of route selection, +to avoid single points of failure and detection, and balancing of local +channels. + +### Routing Example + +Consider four nodes: + + +``` + B + / \\ + / \\ +A C + \\ / + \\ / + D +``` + +Each advertises the following `cltv_expiry_delta` on its end of every +channel: + +1. A: 10 blocks +2. B: 20 blocks +3. C: 30 blocks +4. D: 40 blocks + +C also uses a `min_final_cltv_expiry` of 9 (the default) when requesting +payments. + +Also, each node has a set fee scheme that it uses for each of its +channels: + +1. A: 100 base + 1000 millionths +2. B: 200 base + 2000 millionths +3. C: 300 base + 3000 millionths +4. D: 400 base + 4000 millionths + +The network will see eight `channel_update` messages: + +1. A->B: `cltv_expiry_delta` = 10, `fee_base_msat` = 100, `fee_proportional_millionths` = 1000 +1. A->D: `cltv_expiry_delta` = 10, `fee_base_msat` = 100, `fee_proportional_millionths` = 1000 +1. B->A: `cltv_expiry_delta` = 20, `fee_base_msat` = 200, `fee_proportional_millionths` = 2000 +1. D->A: `cltv_expiry_delta` = 40, `fee_base_msat` = 400, `fee_proportional_millionths` = 4000 +1. B->C: `cltv_expiry_delta` = 20, `fee_base_msat` = 200, `fee_proportional_millionths` = 2000 +1. D->C: `cltv_expiry_delta` = 40, `fee_base_msat` = 400, `fee_proportional_millionths` = 4000 +1. C->B: `cltv_expiry_delta` = 30, `fee_base_msat` = 300, `fee_proportional_millionths` = 3000 +1. C->D: `cltv_expiry_delta` = 30, `fee_base_msat` = 300, `fee_proportional_millionths` = 3000 + +**B->C.** If B were to send 4,999,999 millisatoshi directly to C, it would +neither charge itself a fee nor add its own `cltv_expiry_delta`, so it would +use C's requested `min_final_cltv_expiry` of 9. Presumably it would also add a +_shadow route_ to give an extra CLTV of 42. Additionally, it could add extra +CLTV deltas at other hops, as these values represent a minimum, but chooses not +to do so here, for the sake of simplicity: + + * `amount_msat`: 4999999 + * `cltv_expiry`: current-block-height + 9 + 42 + * `onion_routing_packet`: + * `amt_to_forward` = 4999999 + * `outgoing_cltv_value` = current-block-height + 9 + 42 + +**A->B->C.** If A were to send 4,999,999 millisatoshi to C via B, it needs to +pay B the fee it specified in the B->C `channel_update`, calculated as +per [HTLC Fees](#htlc-fees): + + fee_base_msat + ( amount_to_forward * fee_proportional_millionths / 1000000 ) + + 200 + ( 4999999 * 2000 / 1000000 ) = 10199 + +Similarly, it would need to add B->C's `channel_update` `cltv_expiry` (20), C's +requested `min_final_cltv_expiry` (9), and the cost for the _shadow route_ (42). +Thus, A->B's `update_add_htlc` message would be: + + * `amount_msat`: 5010198 + * `cltv_expiry`: current-block-height + 20 + 9 + 42 + * `onion_routing_packet`: + * `amt_to_forward` = 4999999 + * `outgoing_cltv_value` = current-block-height + 9 + 42 + +B->C's `update_add_htlc` would be the same as B->C's direct payment above. + +**A->D->C.** Finally, if for some reason A chose the more expensive route via D, +A->D's `update_add_htlc` message would be: + + * `amount_msat`: 5020398 + * `cltv_expiry`: current-block-height + 40 + 9 + 42 + * `onion_routing_packet`: + * `amt_to_forward` = 4999999 + * `outgoing_cltv_value` = current-block-height + 9 + 42 + +And D->C's `update_add_htlc` would again be the same as B->C's direct payment +above. + +## References + +1. [RFC 1950 "ZLIB Compressed Data Format Specification version 3.3](https://www.ietf.org/rfc/rfc1950.txt) +2. [Maximum Compression Factor](https://zlib.net/zlib_tech.html) + +![Creative Commons License](https://i.creativecommons.org/l/by/4.0/88x31.png "License CC-BY") +
+This work is licensed under a [Creative Commons Attribution 4.0 International License](http://creativecommons.org/licenses/by/4.0/). +""" diff --git a/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py new file mode 100644 index 000000000..4f6bc8b19 --- /dev/null +++ b/contrib/pyln-spec/bolt7/pyln/spec/bolt7/gen_version.py @@ -0,0 +1,2 @@ +__version__ = "1.0.post137" +__gitversion__ = "9e8e29af9b9a922eb114b2c716205d0772946e56" diff --git a/contrib/pyln-spec/bolt7/requirements.txt b/contrib/pyln-spec/bolt7/requirements.txt new file mode 120000 index 000000000..dc833dd4b --- /dev/null +++ b/contrib/pyln-spec/bolt7/requirements.txt @@ -0,0 +1 @@ +../requirements.txt \ No newline at end of file diff --git a/contrib/pyln-spec/bolt7/setup.py b/contrib/pyln-spec/bolt7/setup.py new file mode 100644 index 000000000..8df5d70b1 --- /dev/null +++ b/contrib/pyln-spec/bolt7/setup.py @@ -0,0 +1,23 @@ +from pyln.spec.bolt7 import __version__, desc +from setuptools import setup +import io + +with io.open('requirements.txt', encoding='utf-8') as f: + requirements = [r for r in f.read().split('\n') if len(r)] + + +def do_setup(boltnum: int, version: str, desc: str): + setup(name='pyln-bolt{}'.format(boltnum), + version=version, + description=desc, + url='http://github.com/ElementsProject/lightning', + author='Rusty Russell', + author_email='rusty@rustcorp.com.au', + license='MIT', + packages=['pyln.spec.bolt{}'.format(boltnum)], + scripts=[], + zip_safe=True, + install_requires=requirements) + + +do_setup(7, __version__, desc) diff --git a/contrib/pyln-proto/tests/test_bolt7.py b/contrib/pyln-spec/bolt7/tests/test_bolt7.py similarity index 90% rename from contrib/pyln-proto/tests/test_bolt7.py rename to contrib/pyln-spec/bolt7/tests/test_bolt7.py index 0ab7decb7..503de5a9d 100644 --- a/contrib/pyln-proto/tests/test_bolt7.py +++ b/contrib/pyln-spec/bolt7/tests/test_bolt7.py @@ -1,6 +1,6 @@ #! /usr/bin/python3 from pyln.proto.message import MessageNamespace -import pyln.proto.message.bolt7 as bolt7 +import pyln.spec.bolt7 as bolt7 # FIXME: more tests diff --git a/contrib/pyln-spec/requirements.txt b/contrib/pyln-spec/requirements.txt new file mode 100644 index 000000000..27820905d --- /dev/null +++ b/contrib/pyln-spec/requirements.txt @@ -0,0 +1 @@ +pyln.proto.message diff --git a/contrib/pyln-proto/pyln/proto/message/bolt4/__init__.py b/contrib/pyln-spec/subinit.py similarity index 57% rename from contrib/pyln-proto/pyln/proto/message/bolt4/__init__.py rename to contrib/pyln-spec/subinit.py index 2ba3aceb6..ee8465380 100644 --- a/contrib/pyln-proto/pyln/proto/message/bolt4/__init__.py +++ b/contrib/pyln-spec/subinit.py @@ -1,12 +1,16 @@ -from .csv import csv +# This is the same __init__.py for all bolt dirs. +from .gen import csv, text, desc +from .gen_version import __version__, __gitversion__ from .bolt import namespace import sys -__version__ = '0.0.1' - __all__ = [ 'csv', + 'text', + 'desc', 'namespace', + '__version__', + '__gitversion__', ] mod = sys.modules[__name__]