pylightning: handle msat fields in JSON more appropriately.

Little point having users handle the postfixes manually, this
translates them, and also allows Millisatoshi to be used wherever an
'int' would be previously.

There are also helpers to create the formatting in a way c-lightning's
JSONRPC will accept.

All standard arithmetic operations with integers work.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

CHANGELOG.md

@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - JSON API: `listchannels` now takes a `source` option to filter by node id.
 - JSON API: New command `paystatus` gives detailed information on `pay` commands.
 - JSON API: `getroute` `riskfactor` argument is simplified; `pay` now defaults to setting it to 10.
+- pylightning: New class 'Millisatoshi' can be used for JSON API, and new '_msat' fields are turned into this on reading.
 
 ### Changed
 

contrib/pylightning/lightning/__init__.py

@@ -1,2 +1,2 @@
-from .lightning import LightningRpc, RpcError
+from .lightning import LightningRpc, RpcError, Millisatoshi
 from .plugin import Plugin, monkey_patch

contrib/pylightning/lightning/lightning.py

@@ -1,3 +1,4 @@
+from decimal import Decimal
 import json
 import logging
 import socket
@@ -13,19 +14,125 @@ class RpcError(ValueError):
         self.error = error
 
 
+class Millisatoshi:
+    """
+    A subtype to represent thousandths of a satoshi.
+
+    Many JSON API fields are expressed in millisatoshis: these automatically get
+    turned into Millisatoshi types.  Converts to and from int.
+    """
+    def __init__(self, v):
+        """
+        Takes either a string ending in 'msat', 'sat', 'btc' or an integer.
+        """
+        if isinstance(v, str):
+            if v.endswith("msat"):
+                self.millisatoshis = int(v[0:-4])
+            elif v.endswith("sat"):
+                self.millisatoshis = Decimal(v[0:-3]) * 1000
+            elif v.endswith("btc"):
+                self.millisatoshis = Decimal(v[0:-3]) * 1000 * 10**8
+            if self.millisatoshis != int(self.millisatoshis):
+                raise ValueError("Millisatoshi must be a whole number")
+        elif isinstance(v, Millisatoshi):
+            self.millisatoshis = v.millisatoshis
+        elif int(v) == v:
+            self.millisatoshis = v
+        else:
+            raise TypeError("Millisatoshi must be string with msat/sat/btc suffix or int")
+
+        if self.millisatoshis < 0:
+            raise ValueError("Millisatoshi must be >= 0")
+
+    def __repr__(self):
+        """
+        Appends the 'msat' as expected for this type.
+        """
+        return str(self.millisatoshis) + "msat"
+
+    def to_satoshi(self):
+        """
+        Return a Decimal representing the number of satoshis
+        """
+        return Decimal(self.millisatoshis) / 1000
+
+    def to_btc(self):
+        """
+        Return a Decimal representing the number of bitcoin
+        """
+        return Decimal(self.millisatoshis) / 1000 / 10**8
+
+    def to_satoshi_str(self):
+        """
+        Return a string of form 1234sat or 1234.567sat.
+        """
+        if self.millisatoshis % 1000:
+            return '{:.3f}sat'.format(self.to_satoshi())
+        else:
+            return '{:.0f}sat'.format(self.to_satoshi())
+
+    def to_btc_str(self):
+        """
+        Return a string of form 12.34567890btc or 12.34567890123btc.
+        """
+        if self.millisatoshis % 1000:
+            return '{:.8f}btc'.format(self.to_btc())
+        else:
+            return '{:.11f}btc'.format(self.to_btc())
+
+    def to_json(self):
+        return self.__repr__()
+
+    def __int__(self):
+        return self.millisatoshis
+
+    def __lt__(self, other):
+        return self.millisatoshis < other.millisatoshis
+
+    def __le__(self, other):
+        return self.millisatoshis <= other.millisatoshis
+
+    def __eq__(self, other):
+        return self.millisatoshis == other.millisatoshis
+
+    def __gt__(self, other):
+        return self.millisatoshis > other.millisatoshis
+
+    def __ge__(self, other):
+        return self.millisatoshis >= other.millisatoshis
+
+    def __add__(self, other):
+        return Millisatoshi(int(self) + int(other))
+
+    def __sub__(self, other):
+        return Millisatoshi(int(self) - int(other))
+
+    def __mul__(self, other):
+        return Millisatoshi(int(self) * other)
+
+    def __truediv__(self, other):
+        return Millisatoshi(int(self) / other)
+
+    def __floordiv__(self, other):
+        return Millisatoshi(int(self) // other)
+
+    def __mod__(self, other):
+        return Millisatoshi(int(self) % other)
+
+
 class UnixDomainSocketRpc(object):
-    def __init__(self, socket_path, executor=None, logger=logging):
+    def __init__(self, socket_path, executor=None, logger=logging, encoder=json.JSONEncoder, decoder=json.JSONDecoder):
         self.socket_path = socket_path
-        self.decoder = json.JSONDecoder()
+        self.encoder = encoder
+        self.decoder = decoder
         self.executor = executor
         self.logger = logger
 
         # Do we require the compatibility mode?
         self._compat = True
 
-    @staticmethod
-    def _writeobj(sock, obj):
-        s = json.dumps(obj)
+    def _writeobj(self, sock, obj):
+        s = json.dumps(obj, cls=self.encoder)
         sock.sendall(bytearray(s, 'UTF-8'))
 
     def _readobj_compat(self, sock, buff=b''):
@@ -128,6 +235,41 @@ class LightningRpc(UnixDomainSocketRpc):
     between calls, but it does not (yet) support concurrent calls.
     """
 
+    class LightningJSONEncoder(json.JSONEncoder):
+        def default(self, o):
+            try:
+                return o.to_json()
+            except NameError:
+                pass
+            return json.JSONEncoder.default(self, o)
+
+    @staticmethod
+    def lightning_json_hook(json_object):
+        return json_object
+
+    @staticmethod
+    def replace_amounts(obj):
+        """
+        Recursively replace _msat fields with appropriate values with Millisatoshi.
+        """
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if k.endswith('msat'):
+                    if isinstance(v, str) and v.endswith('msat'):
+                        obj[k] = Millisatoshi(v)
+                    # Special case for array of msat values
+                    elif isinstance(v, list) and all(isinstance(e, str) and e.endswith('msat') for e in v):
+                        obj[k] = [Millisatoshi(e) for e in v]
+                else:
+                    obj[k] = LightningRpc.replace_amounts(v)
+        elif isinstance(obj, list):
+            obj = [LightningRpc.replace_amounts(e) for e in obj]
+
+        return obj
+
+    def __init__(self, socket_path, executor=None, logger=logging):
+        super().__init__(socket_path, executor, logging, self.LightningJSONEncoder, json.JSONDecoder(object_hook=self.replace_amounts))
+
     def getpeer(self, peer_id, level=None):
         """
         Show peer with {peer_id}, if {level} is set, include {log}s

tests/test_pay.py

@@ -1,5 +1,5 @@
 from fixtures import *  # noqa: F401,F403
-from lightning import RpcError
+from lightning import RpcError, Millisatoshi
 from utils import DEVELOPER, wait_for, only_one, sync_blockheight, SLOW_MACHINE
 
 
@@ -59,6 +59,22 @@ def test_pay(node_factory):
     assert len(payments) == 1 and payments[0]['payment_preimage'] == preimage
 
 
+def test_pay_amounts(node_factory):
+    l1, l2 = node_factory.line_graph(2)
+    inv = l2.rpc.invoice(Millisatoshi("123sat"), 'test_pay_amounts', 'description')['bolt11']
+
+    invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
+
+    assert isinstance(invoice['amount_msat'], Millisatoshi)
+    assert invoice['amount_msat'] == Millisatoshi(123000)
+
+    l1.rpc.pay(inv)
+
+    invoice = only_one(l2.rpc.listinvoices('test_pay_amounts')['invoices'])
+    assert isinstance(invoice['amount_received_msat'], Millisatoshi)
+    assert invoice['amount_received_msat'] >= Millisatoshi(123000)
+
+
 def test_pay_limits(node_factory):
     """Test that we enforce fee max percentage and max delay"""
     l1, l2, l3 = node_factory.line_graph(3, wait_for_announce=True)
@@ -573,7 +589,7 @@ def test_decodepay(node_factory):
     )
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 2500 * 10**11 // 1000000
-    assert b11['amount_msat'] == str(2500 * 10**11 // 1000000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(2500 * 10**11 // 1000000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['description'] == '1 cup coffee'
@@ -607,7 +623,7 @@ def test_decodepay(node_factory):
     )
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(str(20 * 10**11 // 1000) + 'msat')
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -640,7 +656,7 @@ def test_decodepay(node_factory):
     )
     assert b11['currency'] == 'tb'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -672,7 +688,7 @@ def test_decodepay(node_factory):
     b11 = l1.rpc.decodepay('lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqpqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqgqqqqq7qqzqj9n4evl6mr5aj9f58zp6fyjzup6ywn3x6sk8akg5v4tgn2q8g4fhx05wf6juaxu9760yp46454gpg5mtzgerlzezqcqvjnhjh8z3g2qqdhhwkj', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -715,7 +731,7 @@ def test_decodepay(node_factory):
     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -742,7 +758,7 @@ def test_decodepay(node_factory):
     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -769,7 +785,7 @@ def test_decodepay(node_factory):
     b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon')
     assert b11['currency'] == 'bc'
     assert b11['msatoshi'] == 20 * 10**11 // 1000
-    assert b11['amount_msat'] == str(20 * 10**11 // 1000) + 'msat'
+    assert b11['amount_msat'] == Millisatoshi(20 * 10**11 // 1000)
     assert b11['created_at'] == 1496314658
     assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102'
     assert b11['expiry'] == 3600
@@ -1016,10 +1032,10 @@ def test_forward_pad_fees_and_cltv(node_factory, bitcoind):
 
     # Modify so we overpay, overdo the cltv.
     route[0]['msatoshi'] += 2000
-    route[0]['amount_msat'] = str(route[0]['msatoshi']) + 'msat'
+    route[0]['amount_msat'] = Millisatoshi(route[0]['msatoshi'])
     route[0]['delay'] += 20
     route[1]['msatoshi'] += 1000
-    route[1]['amount_msat'] = str(route[1]['msatoshi']) + 'msat'
+    route[1]['amount_msat'] = Millisatoshi(route[1]['msatoshi'])
     route[1]['delay'] += 10
 
     # This should work.