make our channels private, and put routing hints in invoices we create

electrum/lnbase.py

@@ -12,6 +12,7 @@ import time
 import hashlib
 import hmac
 from functools import partial
+from typing import List
 
 import cryptography.hazmat.primitives.ciphers.aead as AEAD
 import aiorpcx
@@ -31,6 +32,7 @@ from .lnutil import (Outpoint, ChannelConfig, LocalState,
                      funding_output_script, get_ecdh, get_per_commitment_secret_from_seed,
                      secret_to_pubkey, LNPeerAddr, PaymentFailure,
                      LOCAL, REMOTE, HTLCOwner, generate_keypair, LnKeyFamily)
+from .lnrouter import NotFoundChanAnnouncementForUpdate, RouteEdge
 
 
 def channel_id_from_funding_tx(funding_txid, funding_index):
@@ -443,7 +445,16 @@ class Peer(PrintError):
         pass
 
     def on_channel_update(self, payload):
-        self.channel_db.on_channel_update(payload)
+        try:
+            self.channel_db.on_channel_update(payload)
+        except NotFoundChanAnnouncementForUpdate:
+            # If it's for a direct channel with this peer, save it in chan.
+            # Note that this is prone to a race.. we might not have a short_channel_id
+            # associated with the channel in some cases
+            short_channel_id = payload['short_channel_id']
+            for chan in self.channels.values():
+                if chan.short_channel_id_predicted == short_channel_id:
+                    chan.pending_channel_update_message = payload
 
     def on_channel_announcement(self, payload):
         self.channel_db.on_channel_announcement(payload)
@@ -550,7 +561,7 @@ class Peer(PrintError):
             first_per_commitment_point=per_commitment_point_first,
             to_self_delay=local_config.to_self_delay,
             max_htlc_value_in_flight_msat=local_config.max_htlc_value_in_flight_msat,
-            channel_flags=0x01, # publicly announcing channel
+            channel_flags=0x00,  # not willing to announce channel
             channel_reserve_satoshis=546
         )
         self.send_message(msg)
@@ -833,6 +844,9 @@ class Peer(PrintError):
         Runs on the Network thread.
         """
         if not chan.local_state.was_announced and funding_tx_depth >= 6:
+            # don't announce our channels
+            # FIXME should this be a field in chan.local_state maybe?
+            return
             chan.local_state=chan.local_state._replace(was_announced=True)
             coro = self.handle_announcements(chan)
             self.lnworker.save_channel(chan)
@@ -887,25 +901,39 @@ class Peer(PrintError):
         chan.set_state("OPEN")
         self.network.trigger_callback('channel', chan)
         # add channel to database
-        node_ids = [self.pubkey, self.lnworker.node_keypair.pubkey]
+        pubkey_ours = self.lnworker.node_keypair.pubkey
+        pubkey_theirs = self.pubkey
+        node_ids = [pubkey_theirs, pubkey_ours]
         bitcoin_keys = [chan.local_config.multisig_key.pubkey, chan.remote_config.multisig_key.pubkey]
         sorted_node_ids = list(sorted(node_ids))
         if sorted_node_ids != node_ids:
             node_ids = sorted_node_ids
             bitcoin_keys.reverse()
-        now = int(time.time()).to_bytes(4, byteorder="big")
+        # note: we inject a channel announcement, and a channel update (for outgoing direction)
+        # This is atm needed for
+        # - finding routes
+        # - the ChanAnn is needed so that we can anchor to it a future ChanUpd
+        #   that the remote sends, even if the channel was not announced
+        #   (from BOLT-07: "MAY create a channel_update to communicate the channel
+        #    parameters to the final node, even though the channel has not yet been announced")
         self.channel_db.on_channel_announcement({"short_channel_id": chan.short_channel_id, "node_id_1": node_ids[0], "node_id_2": node_ids[1],
                                                  'chain_hash': constants.net.rev_genesis_bytes(), 'len': b'\x00\x00', 'features': b'',
                                                  'bitcoin_key_1': bitcoin_keys[0], 'bitcoin_key_2': bitcoin_keys[1]},
                                                 trusted=True)
-        self.channel_db.on_channel_update({"short_channel_id": chan.short_channel_id, 'flags': b'\x01', 'cltv_expiry_delta': b'\x90',
+        # only inject outgoing direction:
-                                           'htlc_minimum_msat': b'\x03\xe8', 'fee_base_msat': b'\x03\xe8', 'fee_proportional_millionths': b'\x01',
+        flags = b'\x00' if node_ids[0] == pubkey_ours else b'\x01'
-                                           'chain_hash': constants.net.rev_genesis_bytes(), 'timestamp': now},
+        now = int(time.time()).to_bytes(4, byteorder="big")
-                                          trusted=True)
+        self.channel_db.on_channel_update({"short_channel_id": chan.short_channel_id, 'flags': flags, 'cltv_expiry_delta': b'\x90',
-        self.channel_db.on_channel_update({"short_channel_id": chan.short_channel_id, 'flags': b'\x00', 'cltv_expiry_delta': b'\x90',
                                            'htlc_minimum_msat': b'\x03\xe8', 'fee_base_msat': b'\x03\xe8', 'fee_proportional_millionths': b'\x01',
                                            'chain_hash': constants.net.rev_genesis_bytes(), 'timestamp': now},
                                           trusted=True)
+        # peer may have sent us a channel update for the incoming direction previously
+        # note: if we were offline when the 3rd conf happened, lnd will never send us this channel_update
+        # see https://github.com/lightningnetwork/lnd/issues/1347
+        #self.send_message(gen_msg("query_short_channel_ids", chain_hash=constants.net.rev_genesis_bytes(),
+        #                          len=9, encoded_short_ids=b'\x00'+chan.short_channel_id))
+        if hasattr(chan, 'pending_channel_update_message'):
+            self.on_channel_update(chan.pending_channel_update_message)
 
         self.print_error("CHANNEL OPENING COMPLETED")
 

electrum/lnhtlc.py

@@ -143,6 +143,7 @@ class HTLCStateMachine(PrintError):
         self.funding_outpoint = Outpoint(**decodeAll(state["funding_outpoint"])) if type(state["funding_outpoint"]) is not Outpoint else state["funding_outpoint"]
         self.node_id = maybeDecode("node_id", state["node_id"]) if type(state["node_id"]) is not bytes else state["node_id"]
         self.short_channel_id = maybeDecode("short_channel_id", state["short_channel_id"]) if type(state["short_channel_id"]) is not bytes else state["short_channel_id"]
+        self.short_channel_id_predicted = self.short_channel_id
         self.onion_keys = {int(k): bfh(v) for k,v in state['onion_keys'].items()} if 'onion_keys' in state else {}
 
         # FIXME this is a tx serialised in the custom electrum partial tx format.

electrum/lnrouter.py

@@ -45,6 +45,9 @@ from .lnutil import LN_GLOBAL_FEATURE_BITS, LNPeerAddr
 class UnknownEvenFeatureBits(Exception): pass
 
 
+class NotFoundChanAnnouncementForUpdate(Exception): pass
+
+
 class ChannelInfo(PrintError):
 
     def __init__(self, channel_announcement_payload):
@@ -126,7 +129,7 @@ class ChannelInfo(PrintError):
         else:
             self.policy_node2 = new_policy
 
-    def get_policy_for_node(self, node_id):
+    def get_policy_for_node(self, node_id: bytes) -> 'ChannelInfoDirectedPolicy':
         if node_id == self.node_id_1:
             return self.policy_node1
         elif node_id == self.node_id_2:
@@ -271,7 +274,7 @@ class ChannelDB(JsonDB):
         JsonDB.__init__(self, path)
 
         self.lock = threading.RLock()
-        self._id_to_channel_info = {}
+        self._id_to_channel_info = {}  # type: Dict[bytes, ChannelInfo]
         self._channels_for_node = defaultdict(set)  # node -> set(short_channel_id)
         self.nodes = {}  # node_id -> NodeInfo
         self._recent_peers = []
@@ -340,7 +343,7 @@ class ChannelDB(JsonDB):
         # number of channels
         return len(self._id_to_channel_info)
 
-    def get_channel_info(self, channel_id) -> Optional[ChannelInfo]:
+    def get_channel_info(self, channel_id: bytes) -> Optional[ChannelInfo]:
         return self._id_to_channel_info.get(channel_id, None)
 
     def get_channels_for_node(self, node_id):
@@ -401,7 +404,7 @@ class ChannelDB(JsonDB):
             channel_info = self._id_to_channel_info.get(short_channel_id, None)
         if channel_info is None:
             self.print_error("could not find", short_channel_id)
-            return
+            raise NotFoundChanAnnouncementForUpdate()
         channel_info.on_channel_update(msg_payload, trusted=trusted)
 
     def on_node_announcement(self, msg_payload):

electrum/lnwatcher.py

@@ -24,6 +24,7 @@ class LNWatcher(PrintError):
         path = os.path.join(network.config.path, "watcher_db")
         storage = WalletStorage(path)
         self.addr_sync = AddressSynchronizer(storage)
+        self.addr_sync.diagnostic_name = lambda: 'LnWatcherAS'
         self.addr_sync.start_network(network)
         self.lock = threading.RLock()
         self.watched_addresses = set()

electrum/lnworker.py

@@ -48,8 +48,8 @@ class LNWorker(PrintError):
         self.ln_keystore = self._read_ln_keystore()
         self.node_keypair = generate_keypair(self.ln_keystore, LnKeyFamily.NODE_KEY, 0)
         self.config = network.config
-        self.peers = {}  # pubkey -> Peer
+        self.peers = {}  # type: Dict[bytes, Peer]  # pubkey -> Peer
-        self.channels = {x.channel_id: x for x in map(HTLCStateMachine, wallet.storage.get("channels", []))}
+        self.channels = {x.channel_id: x for x in map(HTLCStateMachine, wallet.storage.get("channels", []))}  # type: Dict[bytes, HTLCStateMachine]
         for c in self.channels.values():
             c.lnwatcher = network.lnwatcher
             c.sweep_address = self.sweep_address
@@ -126,21 +126,19 @@ class LNWorker(PrintError):
 
     def save_short_chan_id(self, chan):
         """
-        Checks if the Funding TX has been mined. If it has save the short channel ID to disk and return the new OpenChannel.
+        Checks if Funding TX has been mined. If it has, save the short channel ID in chan;
-
+        if it's also deep enough, also save to disk.
-        If the Funding TX has not been mined, return None
+        Returns tuple (mined_deep_enough, num_confirmations).
         """
         assert chan.get_state() in ["OPEN", "OPENING"]
-        peer = self.peers[chan.node_id]
         addr_sync = self.network.lnwatcher.addr_sync
         conf = addr_sync.get_tx_height(chan.funding_outpoint.txid).conf
-        if conf >= chan.constraints.funding_txn_minimum_depth:
+        if conf > 0:
             block_height, tx_pos = addr_sync.get_txpos(chan.funding_outpoint.txid)
-            if tx_pos == -1:
+            assert tx_pos >= 0
-                self.print_error('funding tx is not yet SPV verified.. but there are '
+            chan.short_channel_id_predicted = calc_short_channel_id(block_height, tx_pos, chan.funding_outpoint.output_index)
-                                 'already enough confirmations (currently {})'.format(conf))
+        if conf >= chan.constraints.funding_txn_minimum_depth > 0:
-                return False, conf
+            chan.short_channel_id = chan.short_channel_id_predicted
-            chan.short_channel_id = calc_short_channel_id(block_height, tx_pos, chan.funding_outpoint.output_index)
             self.save_channel(chan)
             return True, conf
         return False, conf
@@ -244,6 +242,7 @@ class LNWorker(PrintError):
         if amount_sat is None:
             raise InvoiceError(_("Missing amount"))
         amount_msat = int(amount_sat * 1000)
+        # TODO use 'r' field from invoice
         path = self.network.path_finder.find_path_for_payment(self.node_keypair.pubkey, invoice_pubkey, amount_msat)
         if path is None:
             raise PaymentFailure(_("No path found"))
@@ -263,12 +262,39 @@ class LNWorker(PrintError):
         payment_preimage = os.urandom(32)
         RHASH = sha256(payment_preimage)
         amount_btc = amount_sat/Decimal(COIN) if amount_sat else None
-        pay_req = lnencode(LnAddr(RHASH, amount_btc, tags=[('d', message)]), self.node_keypair.privkey)
+        routing_hints = self._calc_routing_hints_for_invoice(amount_sat)
+        pay_req = lnencode(LnAddr(RHASH, amount_btc, tags=[('d', message)]+routing_hints),
+                           self.node_keypair.privkey)
         self.invoices[bh2u(payment_preimage)] = pay_req
         self.wallet.storage.put('lightning_invoices', self.invoices)
         self.wallet.storage.write()
         return pay_req
 
+    def _calc_routing_hints_for_invoice(self, amount_sat):
+        """calculate routing hints (BOLT-11 'r' field)"""
+        routing_hints = []
+        with self.lock:
+            channels = list(self.channels.values())
+        # note: currently we add *all* our channels; but this might be a privacy leak?
+        for chan in channels:
+            # check channel is open
+            if chan.get_state() != "OPEN": continue
+            # check channel has sufficient balance
+            # FIXME because of on-chain fees of ctx, this check is insufficient
+            if amount_sat and chan.balance(REMOTE) // 1000 < amount_sat: continue
+            chan_id = chan.short_channel_id
+            assert type(chan_id) is bytes, chan_id
+            channel_info = self.channel_db.get_channel_info(chan_id)
+            if not channel_info: continue
+            policy = channel_info.get_policy_for_node(chan.node_id)
+            if not policy: continue
+            routing_hints.append(('r', [(chan.node_id,
+                                         chan_id,
+                                         policy.fee_base_msat,
+                                         policy.fee_proportional_millionths,
+                                         policy.cltv_expiry_delta)]))
+        return routing_hints
+
     def delete_invoice(self, payreq_key):
         try:
             del self.invoices[payreq_key]