diff --git a/electrum/channel_db.py b/electrum/channel_db.py index 66dec951e..f0da99897 100644 --- a/electrum/channel_db.py +++ b/electrum/channel_db.py @@ -38,7 +38,8 @@ from .sql_db import SqlDB, sql from . import constants from .util import bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits from .logging import Logger -from .lnutil import LN_GLOBAL_FEATURES_KNOWN_SET, LNPeerAddr, format_short_channel_id, ShortChannelID +from .lnutil import (LNPeerAddr, format_short_channel_id, ShortChannelID, + validate_features, IncompatibleOrInsaneFeatures) from .lnverifier import LNChannelVerifier, verify_sig_for_channel_update from .lnmsg import decode_msg @@ -47,15 +48,6 @@ if TYPE_CHECKING: from .lnchannel import Channel -class UnknownEvenFeatureBits(Exception): pass - -def validate_features(features : int): - enabled_features = list_enabled_bits(features) - for fbit in enabled_features: - if (1 << fbit) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0: - raise UnknownEvenFeatureBits() - - FLAG_DISABLE = 1 << 1 FLAG_DIRECTION = 1 << 0 @@ -102,14 +94,14 @@ class Policy(NamedTuple): def from_msg(payload: dict) -> 'Policy': return Policy( key = payload['short_channel_id'] + payload['start_node'], - cltv_expiry_delta = int.from_bytes(payload['cltv_expiry_delta'], "big"), - htlc_minimum_msat = int.from_bytes(payload['htlc_minimum_msat'], "big"), - htlc_maximum_msat = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None, - fee_base_msat = int.from_bytes(payload['fee_base_msat'], "big"), - fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"), + cltv_expiry_delta = payload['cltv_expiry_delta'], + htlc_minimum_msat = payload['htlc_minimum_msat'], + htlc_maximum_msat = payload.get('htlc_maximum_msat', None), + fee_base_msat = payload['fee_base_msat'], + fee_proportional_millionths = payload['fee_proportional_millionths'], message_flags = int.from_bytes(payload['message_flags'], "big"), channel_flags = int.from_bytes(payload['channel_flags'], "big"), - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'], ) @staticmethod @@ -154,7 +146,7 @@ class NodeInfo(NamedTuple): alias = alias.decode('utf8') except: alias = '' - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias) return node_info, peer_addrs @@ -321,11 +313,12 @@ class ChannelDB(SqlDB): return ret # note: currently channel announcements are trusted by default (trusted=True); - # they are not verified. Verifying them would make the gossip sync + # they are not SPV-verified. Verifying them would make the gossip sync # even slower; especially as servers will start throttling us. # It would probably put significant strain on servers if all clients # verified the complete gossip. def add_channel_announcement(self, msg_payloads, *, trusted=True): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] added = 0 @@ -338,8 +331,8 @@ class ChannelDB(SqlDB): continue try: channel_info = ChannelInfo.from_msg(msg) - except UnknownEvenFeatureBits: - self.logger.info("unknown feature bits") + except IncompatibleOrInsaneFeatures as e: + self.logger.info(f"unknown or insane feature bits: {e!r}") continue if trusted: added += 1 @@ -353,7 +346,7 @@ class ChannelDB(SqlDB): def add_verified_channel_info(self, msg: dict, *, capacity_sat: int = None) -> None: try: channel_info = ChannelInfo.from_msg(msg) - except UnknownEvenFeatureBits: + except IncompatibleOrInsaneFeatures: return channel_info = channel_info._replace(capacity_sat=capacity_sat) with self.lock: @@ -392,7 +385,7 @@ class ChannelDB(SqlDB): now = int(time.time()) for payload in payloads: short_channel_id = ShortChannelID(payload['short_channel_id']) - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] if max_age and now - timestamp > max_age: expired.append(payload) continue @@ -407,7 +400,7 @@ class ChannelDB(SqlDB): known.append(payload) # compare updates to existing database entries for payload in known: - timestamp = int.from_bytes(payload['timestamp'], "big") + timestamp = payload['timestamp'] start_node = payload['start_node'] short_channel_id = ShortChannelID(payload['short_channel_id']) key = (start_node, short_channel_id) @@ -499,13 +492,14 @@ class ChannelDB(SqlDB): raise Exception(f'failed verifying channel update for {short_channel_id}') def add_node_announcement(self, msg_payloads): + # note: signatures have already been verified. if type(msg_payloads) is dict: msg_payloads = [msg_payloads] new_nodes = {} for msg_payload in msg_payloads: try: node_info, node_addresses = NodeInfo.from_msg(msg_payload) - except UnknownEvenFeatureBits: + except IncompatibleOrInsaneFeatures: continue node_id = node_info.node_id # Ignore node if it has no associated channel (DoS protection) @@ -599,11 +593,17 @@ class ChannelDB(SqlDB): self._recent_peers = sorted_node_ids[:self.NUM_MAX_RECENT_PEERS] c.execute("""SELECT * FROM channel_info""") for short_channel_id, msg in c: - ci = ChannelInfo.from_raw_msg(msg) + try: + ci = ChannelInfo.from_raw_msg(msg) + except IncompatibleOrInsaneFeatures: + continue self._channels[ShortChannelID.normalize(short_channel_id)] = ci c.execute("""SELECT * FROM node_info""") for node_id, msg in c: - node_info, node_addresses = NodeInfo.from_raw_msg(msg) + try: + node_info, node_addresses = NodeInfo.from_raw_msg(msg) + except IncompatibleOrInsaneFeatures: + continue # don't load node_addresses because they dont have timestamps self._nodes[node_id] = node_info c.execute("""SELECT * FROM policy""") @@ -671,7 +671,7 @@ class ChannelDB(SqlDB): return now = int(time.time()) remote_update_decoded = decode_msg(remote_update_raw)[1] - remote_update_decoded['timestamp'] = now.to_bytes(4, byteorder="big") + remote_update_decoded['timestamp'] = now remote_update_decoded['start_node'] = node_id return Policy.from_msg(remote_update_decoded) elif node_id == chan.get_local_pubkey(): # outgoing direction (from us) diff --git a/electrum/lightning.json b/electrum/lightning.json deleted file mode 100644 index d6794e67c..000000000 --- a/electrum/lightning.json +++ /dev/null @@ -1,903 +0,0 @@ -{ - "init": { - "type": "16", - "payload": { - "gflen": { - "position": "0", - "length": "2" - }, - "globalfeatures": { - "position": "2", - "length": "gflen" - }, - "lflen": { - "position": "2+gflen", - "length": "2" - }, - "localfeatures": { - "position": "4+gflen", - "length": "lflen" - } - } - }, - "error": { - "type": "17", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "data": { - "position": "34", - "length": "len" - } - } - }, - "ping": { - "type": "18", - "payload": { - "num_pong_bytes": { - "position": "0", - "length": "2" - }, - "byteslen": { - "position": "2", - "length": "2" - }, - "ignored": { - "position": "4", - "length": "byteslen" - } - } - }, - "pong": { - "type": "19", - "payload": { - "byteslen": { - "position": "0", - "length": "2" - }, - "ignored": { - "position": "2", - "length": "byteslen" - } - } - }, - "open_channel": { - "type": "32", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "temporary_channel_id": { - "position": "32", - "length": "32" - }, - "funding_satoshis": { - "position": "64", - "length": "8" - }, - "push_msat": { - "position": "72", - "length": "8" - }, - "dust_limit_satoshis": { - "position": "80", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "88", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "96", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "104", - "length": "8" - }, - "feerate_per_kw": { - "position": "112", - "length": "4" - }, - "to_self_delay": { - "position": "116", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "118", - "length": "2" - }, - "funding_pubkey": { - "position": "120", - "length": "33" - }, - "revocation_basepoint": { - "position": "153", - "length": "33" - }, - "payment_basepoint": { - "position": "186", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "219", - "length": "33" - }, - "htlc_basepoint": { - "position": "252", - "length": "33" - }, - "first_per_commitment_point": { - "position": "285", - "length": "33" - }, - "channel_flags": { - "position": "318", - "length": "1" - }, - "shutdown_len": { - "position": "319", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "321", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "accept_channel": { - "type": "33", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "dust_limit_satoshis": { - "position": "32", - "length": "8" - }, - "max_htlc_value_in_flight_msat": { - "position": "40", - "length": "8" - }, - "channel_reserve_satoshis": { - "position": "48", - "length": "8" - }, - "htlc_minimum_msat": { - "position": "56", - "length": "8" - }, - "minimum_depth": { - "position": "64", - "length": "4" - }, - "to_self_delay": { - "position": "68", - "length": "2" - }, - "max_accepted_htlcs": { - "position": "70", - "length": "2" - }, - "funding_pubkey": { - "position": "72", - "length": "33" - }, - "revocation_basepoint": { - "position": "105", - "length": "33" - }, - "payment_basepoint": { - "position": "138", - "length": "33" - }, - "delayed_payment_basepoint": { - "position": "171", - "length": "33" - }, - "htlc_basepoint": { - "position": "204", - "length": "33" - }, - "first_per_commitment_point": { - "position": "237", - "length": "33" - }, - "shutdown_len": { - "position": "270", - "length": "2", - "feature": "option_upfront_shutdown_script" - }, - "shutdown_scriptpubkey": { - "position": "272", - "length": "shutdown_len", - "feature": "option_upfront_shutdown_script" - } - } - }, - "funding_created": { - "type": "34", - "payload": { - "temporary_channel_id": { - "position": "0", - "length": "32" - }, - "funding_txid": { - "position": "32", - "length": "32" - }, - "funding_output_index": { - "position": "64", - "length": "2" - }, - "signature": { - "position": "66", - "length": "64" - } - } - }, - "funding_signed": { - "type": "35", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - } - } - }, - "funding_locked": { - "type": "36", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_per_commitment_point": { - "position": "32", - "length": "33" - } - } - }, - "shutdown": { - "type": "38", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "scriptpubkey": { - "position": "34", - "length": "len" - } - } - }, - "closing_signed": { - "type": "39", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "fee_satoshis": { - "position": "32", - "length": "8" - }, - "signature": { - "position": "40", - "length": "64" - } - } - }, - "update_add_htlc": { - "type": "128", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "amount_msat": { - "position": "40", - "length": "8" - }, - "payment_hash": { - "position": "48", - "length": "32" - }, - "cltv_expiry": { - "position": "80", - "length": "4" - }, - "onion_routing_packet": { - "position": "84", - "length": "1366" - } - } - }, - "update_fulfill_htlc": { - "type": "130", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "payment_preimage": { - "position": "40", - "length": "32" - } - } - }, - "update_fail_htlc": { - "type": "131", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "len": { - "position": "40", - "length": "2" - }, - "reason": { - "position": "42", - "length": "len" - } - } - }, - "update_fail_malformed_htlc": { - "type": "135", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "id": { - "position": "32", - "length": "8" - }, - "sha256_of_onion": { - "position": "40", - "length": "32" - }, - "failure_code": { - "position": "72", - "length": "2" - } - } - }, - "commitment_signed": { - "type": "132", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "signature": { - "position": "32", - "length": "64" - }, - "num_htlcs": { - "position": "96", - "length": "2" - }, - "htlc_signature": { - "position": "98", - "length": "num_htlcs*64" - } - } - }, - "revoke_and_ack": { - "type": "133", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "per_commitment_secret": { - "position": "32", - "length": "32" - }, - "next_per_commitment_point": { - "position": "64", - "length": "33" - } - } - }, - "update_fee": { - "type": "134", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "feerate_per_kw": { - "position": "32", - "length": "4" - } - } - }, - "channel_reestablish": { - "type": "136", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "next_local_commitment_number": { - "position": "32", - "length": "8" - }, - "next_remote_revocation_number": { - "position": "40", - "length": "8" - }, - "your_last_per_commitment_secret": { - "position": "48", - "length": "32", - "feature": "option_data_loss_protect" - }, - "my_current_per_commitment_point": { - "position": "80", - "length": "33", - "feature": "option_data_loss_protect" - } - } - }, - "invalid_realm": { - "type": "PERM|1", - "payload": {} - }, - "temporary_node_failure": { - "type": "NODE|2", - "payload": {} - }, - "permanent_node_failure": { - "type": "PERM|NODE|2", - "payload": {} - }, - "required_node_feature_missing": { - "type": "PERM|NODE|3", - "payload": {} - }, - "invalid_onion_version": { - "type": "BADONION|PERM|4", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_hmac": { - "type": "BADONION|PERM|5", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "invalid_onion_key": { - "type": "BADONION|PERM|6", - "payload": { - "sha256_of_onion": { - "position": "0", - "length": "32" - } - } - }, - "temporary_channel_failure": { - "type": "UPDATE|7", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "permanent_channel_failure": { - "type": "PERM|8", - "payload": {} - }, - "required_channel_feature_missing": { - "type": "PERM|9", - "payload": {} - }, - "unknown_next_peer": { - "type": "PERM|10", - "payload": {} - }, - "amount_below_minimum": { - "type": "UPDATE|11", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "fee_insufficient": { - "type": "UPDATE|12", - "payload": { - "htlc_msat": { - "position": "0", - "length": "8" - }, - "len": { - "position": "8", - "length": "2" - }, - "channel_update": { - "position": "10", - "length": "len" - } - } - }, - "incorrect_cltv_expiry": { - "type": "UPDATE|13", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - }, - "len": { - "position": "4", - "length": "2" - }, - "channel_update": { - "position": "6", - "length": "len" - } - } - }, - "expiry_too_soon": { - "type": "UPDATE|14", - "payload": { - "len": { - "position": "0", - "length": "2" - }, - "channel_update": { - "position": "2", - "length": "len" - } - } - }, - "unknown_payment_hash": { - "type": "PERM|15", - "payload": {} - }, - "incorrect_payment_amount": { - "type": "PERM|16", - "payload": {} - }, - "final_expiry_too_soon": { - "type": "17", - "payload": {} - }, - "final_incorrect_cltv_expiry": { - "type": "18", - "payload": { - "cltv_expiry": { - "position": "0", - "length": "4" - } - } - }, - "final_incorrect_htlc_amount": { - "type": "19", - "payload": { - "incoming_htlc_amt": { - "position": "0", - "length": "8" - } - } - }, - "channel_disabled": { - "type": "UPDATE|20", - "payload": {} - }, - "expiry_too_far": { - "type": "21", - "payload": {} - }, - "announcement_signatures": { - "type": "259", - "payload": { - "channel_id": { - "position": "0", - "length": "32" - }, - "short_channel_id": { - "position": "32", - "length": "8" - }, - "node_signature": { - "position": "40", - "length": "64" - }, - "bitcoin_signature": { - "position": "104", - "length": "64" - } - } - }, - "channel_announcement": { - "type": "256", - "payload": { - "node_signature_1": { - "position": "0", - "length": "64" - }, - "node_signature_2": { - "position": "64", - "length": "64" - }, - "bitcoin_signature_1": { - "position": "128", - "length": "64" - }, - "bitcoin_signature_2": { - "position": "192", - "length": "64" - }, - "len": { - "position": "256", - "length": "2" - }, - "features": { - "position": "258", - "length": "len" - }, - "chain_hash": { - "position": "258+len", - "length": "32" - }, - "short_channel_id": { - "position": "290+len", - "length": "8" - }, - "node_id_1": { - "position": "298+len", - "length": "33" - }, - "node_id_2": { - "position": "331+len", - "length": "33" - }, - "bitcoin_key_1": { - "position": "364+len", - "length": "33" - }, - "bitcoin_key_2": { - "position": "397+len", - "length": "33" - } - } - }, - "node_announcement": { - "type": "257", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "flen": { - "position": "64", - "length": "2" - }, - "features": { - "position": "66", - "length": "flen" - }, - "timestamp": { - "position": "66+flen", - "length": "4" - }, - "node_id": { - "position": "70+flen", - "length": "33" - }, - "rgb_color": { - "position": "103+flen", - "length": "3" - }, - "alias": { - "position": "106+flen", - "length": "32" - }, - "addrlen": { - "position": "138+flen", - "length": "2" - }, - "addresses": { - "position": "140+flen", - "length": "addrlen" - } - } - }, - "channel_update": { - "type": "258", - "payload": { - "signature": { - "position": "0", - "length": "64" - }, - "chain_hash": { - "position": "64", - "length": "32" - }, - "short_channel_id": { - "position": "96", - "length": "8" - }, - "timestamp": { - "position": "104", - "length": "4" - }, - "message_flags": { - "position": "108", - "length": "1" - }, - "channel_flags": { - "position": "109", - "length": "1" - }, - "cltv_expiry_delta": { - "position": "110", - "length": "2" - }, - "htlc_minimum_msat": { - "position": "112", - "length": "8" - }, - "fee_base_msat": { - "position": "120", - "length": "4" - }, - "fee_proportional_millionths": { - "position": "124", - "length": "4" - }, - "htlc_maximum_msat": { - "position": "128", - "length": "8", - "feature": "option_channel_htlc_max" - } - } - }, - "query_short_channel_ids": { - "type": "261", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "len": { - "position": "32", - "length": "2" - }, - "encoded_short_ids": { - "position": "34", - "length": "len" - } - } - }, - "reply_short_channel_ids_end": { - "type": "262", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "complete": { - "position": "32", - "length": "1" - } - } - }, - "query_channel_range": { - "type": "263", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - } - } - }, - "reply_channel_range": { - "type": "264", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_blocknum": { - "position": "32", - "length": "4" - }, - "number_of_blocks": { - "position": "36", - "length": "4" - }, - "complete": { - "position": "40", - "length": "1" - }, - "len": { - "position": "41", - "length": "2" - }, - "encoded_short_ids": { - "position": "43", - "length": "len" - } - } - }, - "gossip_timestamp_filter": { - "type": "265", - "payload": { - "chain_hash": { - "position": "0", - "length": "32" - }, - "first_timestamp": { - "position": "32", - "length": "4" - }, - "timestamp_range": { - "position": "36", - "length": "4" - } - } - } -} diff --git a/electrum/lnaddr.py b/electrum/lnaddr.py index 027cdc578..a10054055 100644 --- a/electrum/lnaddr.py +++ b/electrum/lnaddr.py @@ -141,6 +141,21 @@ def tagged(char, l): def tagged_bytes(char, l): return tagged(char, bitstring.BitArray(l)) +def trim_to_min_length(bits): + """Ensures 'bits' have min number of leading zeroes. + Assumes 'bits' is big-endian, and that it needs to be encoded in 5 bit blocks. + """ + bits = bits[:] # copy + # make sure we can be split into 5 bit blocks + while bits.len % 5 != 0: + bits.prepend('0b0') + # Get minimal length by trimming leading 5 bits at a time. + while bits.startswith('0b00000'): + if len(bits) == 5: + break # v == 0 + bits = bits[5:] + return bits + # Discard trailing bits, convert to bytes. def trim_to_bytes(barr): # Adds a byte if necessary. @@ -155,7 +170,7 @@ def pull_tagged(stream): length = stream.read(5).uint * 32 + stream.read(5).uint return (CHARSET[tag], stream.read(length * 5), stream) -def lnencode(addr, privkey): +def lnencode(addr: 'LnAddr', privkey) -> str: if addr.amount: amount = Decimal(str(addr.amount)) # We can only send down to millisatoshi. @@ -172,16 +187,22 @@ def lnencode(addr, privkey): # Start with the timestamp data = bitstring.pack('uint:35', addr.date) + tags_set = set() + # Payment hash data += tagged_bytes('p', addr.paymenthash) - tags_set = set() + tags_set.add('p') + + if addr.payment_secret is not None: + data += tagged_bytes('s', addr.payment_secret) + tags_set.add('s') for k, v in addr.tags: # BOLT #11: # # A writer MUST NOT include more than one `d`, `h`, `n` or `x` fields, - if k in ('d', 'h', 'n', 'x'): + if k in ('d', 'h', 'n', 'x', 'p', 's'): if k in tags_set: raise ValueError("Duplicate '{}' tag".format(k)) @@ -196,23 +217,23 @@ def lnencode(addr, privkey): elif k == 'd': data += tagged_bytes('d', v.encode()) elif k == 'x': - # Get minimal length by trimming leading 5 bits at a time. - expirybits = bitstring.pack('intbe:64', v)[4:64] - while expirybits.startswith('0b00000'): - if len(expirybits) == 5: - break # v == 0 - expirybits = expirybits[5:] + expirybits = bitstring.pack('intbe:64', v) + expirybits = trim_to_min_length(expirybits) data += tagged('x', expirybits) elif k == 'h': data += tagged_bytes('h', sha256(v.encode('utf-8')).digest()) elif k == 'n': data += tagged_bytes('n', v) elif k == 'c': - # Get minimal length by trimming leading 5 bits at a time. - finalcltvbits = bitstring.pack('intbe:64', v)[4:64] - while finalcltvbits.startswith('0b00000'): - finalcltvbits = finalcltvbits[5:] + finalcltvbits = bitstring.pack('intbe:64', v) + finalcltvbits = trim_to_min_length(finalcltvbits) data += tagged('c', finalcltvbits) + elif k == '9': + if v == 0: + continue + feature_bits = bitstring.BitArray(uint=v, length=v.bit_length()) + feature_bits = trim_to_min_length(feature_bits) + data += tagged('9', feature_bits) else: # FIXME: Support unknown tags? raise ValueError("Unknown tag {}".format(k)) @@ -239,15 +260,17 @@ def lnencode(addr, privkey): return bech32_encode(hrp, bitarray_to_u5(data)) class LnAddr(object): - def __init__(self, paymenthash: bytes = None, amount=None, currency=None, tags=None, date=None): + def __init__(self, *, paymenthash: bytes = None, amount=None, currency=None, tags=None, date=None, + payment_secret: bytes = None): self.date = int(time.time()) if not date else int(date) self.tags = [] if not tags else tags self.unknown_tags = [] self.paymenthash = paymenthash + self.payment_secret = payment_secret self.signature = None self.pubkey = None self.currency = constants.net.SEGWIT_HRP if currency is None else currency - self.amount = amount + self.amount = amount # in bitcoins self._min_final_cltv_expiry = 9 def __str__(self): @@ -383,14 +406,28 @@ def lndecode(invoice: str, *, verbose=False, expected_hrp=None) -> LnAddr: continue addr.paymenthash = trim_to_bytes(tagdata) + elif tag == 's': + if data_length != 52: + addr.unknown_tags.append((tag, tagdata)) + continue + addr.payment_secret = trim_to_bytes(tagdata) + elif tag == 'n': if data_length != 53: addr.unknown_tags.append((tag, tagdata)) continue pubkeybytes = trim_to_bytes(tagdata) addr.pubkey = pubkeybytes + elif tag == 'c': addr._min_final_cltv_expiry = tagdata.int + + elif tag == '9': + features = tagdata.uint + addr.tags.append(('9', features)) + from .lnutil import validate_features + validate_features(features) + else: addr.unknown_tags.append((tag, tagdata)) diff --git a/electrum/lnchannel.py b/electrum/lnchannel.py index d38bd4824..4fecf6a03 100644 --- a/electrum/lnchannel.py +++ b/electrum/lnchannel.py @@ -218,13 +218,13 @@ class Channel(Logger): short_channel_id=self.short_channel_id, channel_flags=channel_flags, message_flags=b'\x01', - cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA.to_bytes(2, byteorder="big"), - htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat.to_bytes(8, byteorder="big"), - htlc_maximum_msat=htlc_maximum_msat.to_bytes(8, byteorder="big"), - fee_base_msat=lnutil.OUR_FEE_BASE_MSAT.to_bytes(4, byteorder="big"), - fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS.to_bytes(4, byteorder="big"), + cltv_expiry_delta=lnutil.NBLOCK_OUR_CLTV_EXPIRY_DELTA, + htlc_minimum_msat=self.config[REMOTE].htlc_minimum_msat, + htlc_maximum_msat=htlc_maximum_msat, + fee_base_msat=lnutil.OUR_FEE_BASE_MSAT, + fee_proportional_millionths=lnutil.OUR_FEE_PROPORTIONAL_MILLIONTHS, chain_hash=constants.net.rev_genesis_bytes(), - timestamp=now.to_bytes(4, byteorder="big"), + timestamp=now, ) sighash = sha256d(chan_upd[2 + 64:]) sig = ecc.ECPrivkey(self.lnworker.node_keypair.privkey).sign(sighash, ecc.sig_string_from_r_and_s) @@ -249,7 +249,8 @@ class Channel(Logger): node_ids = sorted_node_ids bitcoin_keys.reverse() - chan_ann = encode_msg("channel_announcement", + chan_ann = encode_msg( + "channel_announcement", len=0, features=b'', chain_hash=constants.net.rev_genesis_bytes(), @@ -257,7 +258,7 @@ class Channel(Logger): node_id_1=node_ids[0], node_id_2=node_ids[1], bitcoin_key_1=bitcoin_keys[0], - bitcoin_key_2=bitcoin_keys[1] + bitcoin_key_2=bitcoin_keys[1], ) self._chan_ann_without_sigs = chan_ann diff --git a/electrum/lnmsg.py b/electrum/lnmsg.py index a755756b4..0eb0019ac 100644 --- a/electrum/lnmsg.py +++ b/electrum/lnmsg.py @@ -1,153 +1,513 @@ -import json import os -from typing import Callable, Tuple +import csv +import io +from typing import Callable, Tuple, Any, Dict, List, Sequence, Union, Optional from collections import OrderedDict -def _eval_length_term(x, ma: dict) -> int: - """ - Evaluate a term of the simple language used - to specify lightning message field lengths. +from .lnutil import OnionFailureCodeMetaFlag - If `x` is an integer, it is returned as is, - otherwise it is treated as a variable and - looked up in `ma`. - If the value in `ma` was no integer, it is - assumed big-endian bytes and decoded. +class MalformedMsg(Exception): pass +class UnknownMsgFieldType(MalformedMsg): pass +class UnexpectedEndOfStream(MalformedMsg): pass +class FieldEncodingNotMinimal(MalformedMsg): pass +class UnknownMandatoryTLVRecordType(MalformedMsg): pass +class MsgTrailingGarbage(MalformedMsg): pass +class MsgInvalidFieldOrder(MalformedMsg): pass +class UnexpectedFieldSizeForEncoder(MalformedMsg): pass - Returns evaluated result as int - """ - try: - x = int(x) - except ValueError: - x = ma[x] + +def _num_remaining_bytes_to_read(fd: io.BytesIO) -> int: + cur_pos = fd.tell() + end_pos = fd.seek(0, io.SEEK_END) + fd.seek(cur_pos) + return end_pos - cur_pos + + +def _assert_can_read_at_least_n_bytes(fd: io.BytesIO, n: int) -> None: + # note: it's faster to read n bytes and then check if we read n, than + # to assert we can read at least n and then read n bytes. + nremaining = _num_remaining_bytes_to_read(fd) + if nremaining < n: + raise UnexpectedEndOfStream(f"wants to read {n} bytes but only {nremaining} bytes left") + + +def write_bigsize_int(i: int) -> bytes: + assert i >= 0, i + if i < 0xfd: + return int.to_bytes(i, length=1, byteorder="big", signed=False) + elif i < 0x1_0000: + return b"\xfd" + int.to_bytes(i, length=2, byteorder="big", signed=False) + elif i < 0x1_0000_0000: + return b"\xfe" + int.to_bytes(i, length=4, byteorder="big", signed=False) + else: + return b"\xff" + int.to_bytes(i, length=8, byteorder="big", signed=False) + + +def read_bigsize_int(fd: io.BytesIO) -> Optional[int]: try: - x = int(x) - except ValueError: - x = int.from_bytes(x, byteorder='big') - return x + first = fd.read(1)[0] + except IndexError: + return None # end of file + if first < 0xfd: + return first + elif first == 0xfd: + buf = fd.read(2) + if len(buf) != 2: + raise UnexpectedEndOfStream() + val = int.from_bytes(buf, byteorder="big", signed=False) + if not (0xfd <= val < 0x1_0000): + raise FieldEncodingNotMinimal() + return val + elif first == 0xfe: + buf = fd.read(4) + if len(buf) != 4: + raise UnexpectedEndOfStream() + val = int.from_bytes(buf, byteorder="big", signed=False) + if not (0x1_0000 <= val < 0x1_0000_0000): + raise FieldEncodingNotMinimal() + return val + elif first == 0xff: + buf = fd.read(8) + if len(buf) != 8: + raise UnexpectedEndOfStream() + val = int.from_bytes(buf, byteorder="big", signed=False) + if not (0x1_0000_0000 <= val): + raise FieldEncodingNotMinimal() + return val + raise Exception() -def _eval_exp_with_ctx(exp, ctx: dict) -> int: - """ - Evaluate simple mathematical expression given - in `exp` with context (variables assigned) - from the dict `ctx`. - Returns evaluated result as int - """ - exp = str(exp) - if "*" in exp: - assert "+" not in exp - result = 1 - for term in exp.split("*"): - result *= _eval_length_term(term, ctx) - return result - return sum(_eval_length_term(x, ctx) for x in exp.split("+")) - -def _make_handler(msg_name: str, v: dict) -> Callable[[bytes], Tuple[str, dict]]: - """ - Generate a message handler function (taking bytes) - for message type `msg_name` with specification `v` +# TODO: maybe if field_type is not "byte", we could return a list of type_len sized chunks? +# if field_type is a numeric, we could return a list of ints? +def _read_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str]) -> Union[bytes, int]: + if not fd: raise Exception() + if isinstance(count, int): + assert count >= 0, f"{count!r} must be non-neg int" + elif count == "...": + pass + else: + raise Exception(f"unexpected field count: {count!r}") + if count == 0: + return b"" + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type in ('u8', 'u16', 'u32', 'u64'): + if field_type == 'u8': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + else: + assert field_type == 'u64' + type_len = 8 + assert count == 1, count + buf = fd.read(type_len) + if len(buf) != type_len: + raise UnexpectedEndOfStream() + return int.from_bytes(buf, byteorder="big", signed=False) + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 + assert count == 1, count + raw = fd.read(type_len) + if len(raw) > 0 and raw[0] == 0x00: + raise FieldEncodingNotMinimal() + return int.from_bytes(raw, byteorder="big", signed=False) + elif field_type == 'varint': + assert count == 1, count + val = read_bigsize_int(fd) + if val is None: + raise UnexpectedEndOfStream() + return val + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + + if count == "...": + total_len = -1 # read all + else: + if type_len is None: + raise UnknownMsgFieldType(f"unknown field type: {field_type!r}") + total_len = count * type_len + + buf = fd.read(total_len) + if total_len >= 0 and len(buf) != total_len: + raise UnexpectedEndOfStream() + return buf + - Check lib/lightning.json, `msg_name` could be 'init', - and `v` could be +# TODO: maybe for "value" we could accept a list with len "count" of appropriate items +def _write_field(*, fd: io.BytesIO, field_type: str, count: Union[int, str], + value: Union[bytes, int]) -> None: + if not fd: raise Exception() + if isinstance(count, int): + assert count >= 0, f"{count!r} must be non-neg int" + elif count == "...": + pass + else: + raise Exception(f"unexpected field count: {count!r}") + if count == 0: + return + type_len = None + if field_type == 'byte': + type_len = 1 + elif field_type == 'u8': + type_len = 1 + elif field_type == 'u16': + type_len = 2 + elif field_type == 'u32': + type_len = 4 + elif field_type == 'u64': + type_len = 8 + elif field_type in ('tu16', 'tu32', 'tu64'): + if field_type == 'tu16': + type_len = 2 + elif field_type == 'tu32': + type_len = 4 + else: + assert field_type == 'tu64' + type_len = 8 + assert count == 1, count + if isinstance(value, int): + value = int.to_bytes(value, length=type_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + while len(value) > 0 and value[0] == 0x00: + value = value[1:] + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return + elif field_type == 'varint': + assert count == 1, count + if isinstance(value, int): + value = write_bigsize_int(value) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") + return + elif field_type == 'chain_hash': + type_len = 32 + elif field_type == 'channel_id': + type_len = 32 + elif field_type == 'sha256': + type_len = 32 + elif field_type == 'signature': + type_len = 64 + elif field_type == 'point': + type_len = 33 + elif field_type == 'short_channel_id': + type_len = 8 + total_len = -1 + if count != "...": + if type_len is None: + raise UnknownMsgFieldType(f"unknown field type: {field_type!r}") + total_len = count * type_len + if isinstance(value, int) and (count == 1 or field_type == 'byte'): + value = int.to_bytes(value, length=total_len, byteorder="big", signed=False) + if not isinstance(value, (bytes, bytearray)): + raise Exception(f"can only write bytes into fd. got: {value!r}") + if count != "..." and total_len != len(value): + raise UnexpectedFieldSizeForEncoder(f"expected: {total_len}, got {len(value)}") + nbytes_written = fd.write(value) + if nbytes_written != len(value): + raise Exception(f"tried to write {len(value)} bytes, but only wrote {nbytes_written}!?") - { type: 16, payload: { 'gflen': ..., ... }, ... } - Returns function taking bytes +def _read_tlv_record(*, fd: io.BytesIO) -> Tuple[int, bytes]: + if not fd: raise Exception() + tlv_type = _read_field(fd=fd, field_type="varint", count=1) + tlv_len = _read_field(fd=fd, field_type="varint", count=1) + tlv_val = _read_field(fd=fd, field_type="byte", count=tlv_len) + return tlv_type, tlv_val + + +def _write_tlv_record(*, fd: io.BytesIO, tlv_type: int, tlv_val: bytes) -> None: + if not fd: raise Exception() + tlv_len = len(tlv_val) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_type) + _write_field(fd=fd, field_type="varint", count=1, value=tlv_len) + _write_field(fd=fd, field_type="byte", count=tlv_len, value=tlv_val) + + +def _resolve_field_count(field_count_str: str, *, vars_dict: dict, allow_any=False) -> Union[int, str]: + """Returns an evaluated field count, typically an int. + If allow_any is True, the return value can be a str with value=="...". """ - def handler(data: bytes) -> Tuple[str, dict]: - ma = {} # map of field name -> field data; after parsing msg - pos = 0 - for fieldname in v["payload"]: - poslenMap = v["payload"][fieldname] - if "feature" in poslenMap and pos == len(data): - continue - #assert pos == _eval_exp_with_ctx(poslenMap["position"], ma) # this assert is expensive... - length = poslenMap["length"] - length = _eval_exp_with_ctx(length, ma) - ma[fieldname] = data[pos:pos+length] - pos += length - # BOLT-01: "MUST ignore any additional data within a message beyond the length that it expects for that type." - assert pos <= len(data), (msg_name, pos, len(data)) - return msg_name, ma - return handler + if field_count_str == "": + field_count = 1 + elif field_count_str == "...": + if not allow_any: + raise Exception("field count is '...' but allow_any is False") + return field_count_str + else: + try: + field_count = int(field_count_str) + except ValueError: + field_count = vars_dict[field_count_str] + if isinstance(field_count, (bytes, bytearray)): + field_count = int.from_bytes(field_count, byteorder="big") + assert isinstance(field_count, int) + return field_count + + +def _parse_msgtype_intvalue_for_onion_wire(value: str) -> int: + msg_type_int = 0 + for component in value.split("|"): + try: + msg_type_int |= int(component) + except ValueError: + msg_type_int |= OnionFailureCodeMetaFlag[component] + return msg_type_int + class LNSerializer: - def __init__(self): - message_types = {} - path = os.path.join(os.path.dirname(__file__), 'lightning.json') - with open(path) as f: - structured = json.loads(f.read(), object_pairs_hook=OrderedDict) - - for msg_name in structured: - v = structured[msg_name] - # these message types are skipped since their types collide - # (for example with pong, which also uses type=19) - # we don't need them yet - if msg_name in ["final_incorrect_cltv_expiry", "final_incorrect_htlc_amount"]: - continue - if len(v["payload"]) == 0: + + def __init__(self, *, for_onion_wire: bool = False): + # TODO msg_type could be 'int' everywhere... + self.msg_scheme_from_type = {} # type: Dict[bytes, List[Sequence[str]]] + self.msg_type_from_name = {} # type: Dict[str, bytes] + + self.in_tlv_stream_get_tlv_record_scheme_from_type = {} # type: Dict[str, Dict[int, List[Sequence[str]]]] + self.in_tlv_stream_get_record_type_from_name = {} # type: Dict[str, Dict[str, int]] + self.in_tlv_stream_get_record_name_from_type = {} # type: Dict[str, Dict[int, str]] + + if for_onion_wire: + path = os.path.join(os.path.dirname(__file__), "lnwire", "onion_wire.csv") + else: + path = os.path.join(os.path.dirname(__file__), "lnwire", "peer_wire.csv") + with open(path, newline='') as f: + csvreader = csv.reader(f) + for row in csvreader: + #print(f">>> {row!r}") + if row[0] == "msgtype": + # msgtype,,[,