From 30f5be26acb182d3f8063c2f3afb09fc9f39433a Mon Sep 17 00:00:00 2001 From: ThomasV Date: Tue, 9 Jun 2020 10:05:23 +0200 Subject: [PATCH 1/4] Remove dependencies: jsonrpcserver, jsonrpcclient --- contrib/requirements/requirements.txt | 2 -- electrum/daemon.py | 41 ++++++++++++++------------- electrum/lnworker.py | 11 +++---- electrum/util.py | 27 ++++++++++++++++++ 4 files changed, 52 insertions(+), 29 deletions(-) diff --git a/contrib/requirements/requirements.txt b/contrib/requirements/requirements.txt index 2a9ff93c0..031fb19eb 100644 --- a/contrib/requirements/requirements.txt +++ b/contrib/requirements/requirements.txt @@ -9,6 +9,4 @@ aiohttp>=3.3.0,<4.0.0 aiohttp_socks>=0.3 certifi bitstring -jsonrpcserver -jsonrpcclient attrs diff --git a/electrum/daemon.py b/electrum/daemon.py index 62c314a09..7d26c8873 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -37,11 +37,8 @@ from concurrent import futures import aiohttp from aiohttp import web, client_exceptions -import jsonrpcclient -import jsonrpcserver -from jsonrpcserver import response -from jsonrpcclient.clients.aiohttp_client import AiohttpClient from aiorpcx import TaskGroup +import json from . import util from .network import Network @@ -107,10 +104,8 @@ def request(config: SimpleConfig, endpoint, args=(), timeout=60): loop = asyncio.get_event_loop() async def request_coroutine(): async with aiohttp.ClientSession(auth=auth) as session: - server = AiohttpClient(session, server_url, timeout=timeout) - f = getattr(server, endpoint) - response = await f(*args) - return response.data.result + c = util.myAiohttpClient(session, server_url) + return await c.request(endpoint, *args) try: fut = asyncio.run_coroutine_threadsafe(request_coroutine(), loop) return fut.result(timeout=timeout) @@ -184,16 +179,22 @@ class AuthenticatedServer(Logger): text='Unauthorized', status=401) except AuthenticationCredentialsInvalid: return web.Response(text='Forbidden', status=403) - request = await request.text() - response = await jsonrpcserver.async_dispatch(request, methods=self.methods) - if isinstance(response, jsonrpcserver.response.ExceptionResponse): - self.logger.error(f"error handling request: {request}", exc_info=response.exc) - # this exposes the error message to the client - response.message = str(response.exc) - if response.wanted: - return web.json_response(response.deserialized(), status=response.http_status) - else: - return web.Response() + try: + request = await request.text() + request = json.loads(request) + method = request['method'] + _id = request['id'] + params = request.get('params', []) + f = getattr(self, method) + assert f in self.methods + except: + return web.Response(text='Invalid Request', status=500) + response = {'id':_id} + try: + response['result'] = await f(*params) + except BaseException as e: + response['error'] = str(e) + return web.json_response(response) class CommandsServer(AuthenticatedServer): @@ -208,7 +209,7 @@ class CommandsServer(AuthenticatedServer): self.port = self.config.get('rpcport', 0) self.app = web.Application() self.app.router.add_post("/", self.handle) - self.methods = jsonrpcserver.methods.Methods() + self.methods = set() self.methods.add(self.ping) self.methods.add(self.gui) self.cmd_runner = Commands(config=self.config, network=self.daemon.network, daemon=self.daemon) @@ -276,7 +277,7 @@ class WatchTowerServer(AuthenticatedServer): self.lnwatcher = network.local_watchtower self.app = web.Application() self.app.router.add_post("/", self.handle) - self.methods = jsonrpcserver.methods.Methods() + self.methods = set() self.methods.add(self.get_ctn) self.methods.add(self.add_sweep_tx) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index 7a8aa3810..d7e718914 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -10,6 +10,7 @@ import time from typing import Optional, Sequence, Tuple, List, Dict, TYPE_CHECKING, NamedTuple, Union, Mapping import threading import socket +import aiohttp import json from datetime import datetime, timezone from functools import partial @@ -25,7 +26,7 @@ from . import constants, util from . import keystore from .util import profiler from .invoices import PR_TYPE_LN, PR_UNPAID, PR_EXPIRED, PR_PAID, PR_INFLIGHT, PR_FAILED, PR_ROUTING, LNInvoice, LN_EXPIRY_NEVER -from .util import NetworkRetryManager +from .util import NetworkRetryManager, myAiohttpClient from .lnutil import LN_MAX_FUNDING_SAT from .keystore import BIP32_KeyStore from .bitcoin import COIN @@ -525,12 +526,6 @@ class LNWallet(LNWorker): @ignore_exceptions @log_exceptions async def sync_with_remote_watchtower(self): - import aiohttp - from jsonrpcclient.clients.aiohttp_client import AiohttpClient - class myAiohttpClient(AiohttpClient): - async def request(self, *args, **kwargs): - r = await super().request(*args, **kwargs) - return r.data.result while True: await asyncio.sleep(5) watchtower_url = self.config.get('watchtower_url') @@ -539,6 +534,8 @@ class LNWallet(LNWorker): try: async with make_aiohttp_session(proxy=self.network.proxy) as session: watchtower = myAiohttpClient(session, watchtower_url) + watchtower.add_method('get_ctn') + watchtower.add_method('add_sweep_tx') for chan in self.channels.values(): await self.sync_channel_with_watchtower(chan, watchtower) except aiohttp.client_exceptions.ClientConnectorError: diff --git a/electrum/util.py b/electrum/util.py index 36a8af1d1..b8401543b 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -1368,3 +1368,30 @@ class MySocksProxy(aiorpcx.SOCKSProxy): else: raise NotImplementedError # http proxy not available with aiorpcx return ret + + +class myAiohttpClient: + + def __init__(self, session, url): + self.session = session + self.url = url + + async def request(self, endpoint, *args): + data = '{"jsonrpc": "2.0", "id":"0", "method":"%s", "params": %s }' %(endpoint, json.dumps(args)) + async with self.session.post(self.url, data=data) as resp: + if resp.status == 200: + r = await resp.json() + result = r.get('result') + error = r.get('error') + if error: + return 'Error: ' + str(error) + else: + return result + else: + text = await resp.text() + return 'Error: ' + str(text) + + def add_method(self, endpoint): + async def coro(*args): + return await self.request(endpoint, *args) + setattr(self, endpoint, coro) From 50f705ee4638cbc795e4aa26c1e99d4f421e7c05 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 9 Jun 2020 17:45:04 +0200 Subject: [PATCH 2/4] fix json-rpc interface (when not using CLI) --- electrum/daemon.py | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/electrum/daemon.py b/electrum/daemon.py index 7d26c8873..e2a227476 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -29,16 +29,16 @@ import time import traceback import sys import threading -from typing import Dict, Optional, Tuple, Iterable +from typing import Dict, Optional, Tuple, Iterable, Callable, Union, Sequence, Mapping from base64 import b64decode, b64encode from collections import defaultdict import concurrent from concurrent import futures +import json import aiohttp from aiohttp import web, client_exceptions from aiorpcx import TaskGroup -import json from . import util from .network import Network @@ -151,6 +151,11 @@ class AuthenticatedServer(Logger): self.rpc_user = rpc_user self.rpc_password = rpc_password self.auth_lock = asyncio.Lock() + self._methods = {} # type: Dict[str, Callable] + + def register_method(self, f): + assert f.__name__ not in self._methods, f"name collision for {f.__name__}" + self._methods[f.__name__] = f async def authenticate(self, headers): if self.rpc_password == '': @@ -184,15 +189,21 @@ class AuthenticatedServer(Logger): request = json.loads(request) method = request['method'] _id = request['id'] - params = request.get('params', []) - f = getattr(self, method) - assert f in self.methods - except: + params = request.get('params', []) # type: Union[Sequence, Mapping] + if method not in self._methods: + raise Exception(f"attempting to use unregistered method: {method}") + f = self._methods[method] + except Exception as e: + self.logger.exception("invalid request") return web.Response(text='Invalid Request', status=500) - response = {'id':_id} + response = {'id': _id} try: - response['result'] = await f(*params) + if isinstance(params, dict): + response['result'] = await f(**params) + else: + response['result'] = await f(*params) except BaseException as e: + self.logger.exception("internal error while executing RPC") response['error'] = str(e) return web.json_response(response) @@ -209,13 +220,12 @@ class CommandsServer(AuthenticatedServer): self.port = self.config.get('rpcport', 0) self.app = web.Application() self.app.router.add_post("/", self.handle) - self.methods = set() - self.methods.add(self.ping) - self.methods.add(self.gui) + self.register_method(self.ping) + self.register_method(self.gui) self.cmd_runner = Commands(config=self.config, network=self.daemon.network, daemon=self.daemon) for cmdname in known_commands: - self.methods.add(getattr(self.cmd_runner, cmdname)) - self.methods.add(self.run_cmdline) + self.register_method(getattr(self.cmd_runner, cmdname)) + self.register_method(self.run_cmdline) async def run(self): self.runner = web.AppRunner(self.app) @@ -277,9 +287,8 @@ class WatchTowerServer(AuthenticatedServer): self.lnwatcher = network.local_watchtower self.app = web.Application() self.app.router.add_post("/", self.handle) - self.methods = set() - self.methods.add(self.get_ctn) - self.methods.add(self.add_sweep_tx) + self.register_method(self.get_ctn) + self.register_method(self.add_sweep_tx) async def run(self): self.runner = web.AppRunner(self.app) From a32cb7784f55e0ef4abbc009a5116056063103b5 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 9 Jun 2020 17:50:06 +0200 Subject: [PATCH 3/4] myAiohttpClient: add id counter, and rename to JsonRPCClient --- electrum/daemon.py | 2 +- electrum/lnworker.py | 4 ++-- electrum/util.py | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/electrum/daemon.py b/electrum/daemon.py index e2a227476..821a09864 100644 --- a/electrum/daemon.py +++ b/electrum/daemon.py @@ -104,7 +104,7 @@ def request(config: SimpleConfig, endpoint, args=(), timeout=60): loop = asyncio.get_event_loop() async def request_coroutine(): async with aiohttp.ClientSession(auth=auth) as session: - c = util.myAiohttpClient(session, server_url) + c = util.JsonRPCClient(session, server_url) return await c.request(endpoint, *args) try: fut = asyncio.run_coroutine_threadsafe(request_coroutine(), loop) diff --git a/electrum/lnworker.py b/electrum/lnworker.py index d7e718914..02cd671ce 100644 --- a/electrum/lnworker.py +++ b/electrum/lnworker.py @@ -26,7 +26,7 @@ from . import constants, util from . import keystore from .util import profiler from .invoices import PR_TYPE_LN, PR_UNPAID, PR_EXPIRED, PR_PAID, PR_INFLIGHT, PR_FAILED, PR_ROUTING, LNInvoice, LN_EXPIRY_NEVER -from .util import NetworkRetryManager, myAiohttpClient +from .util import NetworkRetryManager, JsonRPCClient from .lnutil import LN_MAX_FUNDING_SAT from .keystore import BIP32_KeyStore from .bitcoin import COIN @@ -533,7 +533,7 @@ class LNWallet(LNWorker): continue try: async with make_aiohttp_session(proxy=self.network.proxy) as session: - watchtower = myAiohttpClient(session, watchtower_url) + watchtower = JsonRPCClient(session, watchtower_url) watchtower.add_method('get_ctn') watchtower.add_method('add_sweep_tx') for chan in self.channels.values(): diff --git a/electrum/util.py b/electrum/util.py index b8401543b..36296ff3c 100644 --- a/electrum/util.py +++ b/electrum/util.py @@ -1370,14 +1370,17 @@ class MySocksProxy(aiorpcx.SOCKSProxy): return ret -class myAiohttpClient: +class JsonRPCClient: - def __init__(self, session, url): + def __init__(self, session: aiohttp.ClientSession, url: str): self.session = session self.url = url + self._id = 0 async def request(self, endpoint, *args): - data = '{"jsonrpc": "2.0", "id":"0", "method":"%s", "params": %s }' %(endpoint, json.dumps(args)) + self._id += 1 + data = ('{"jsonrpc": "2.0", "id":"%d", "method": "%s", "params": %s }' + % (self._id, endpoint, json.dumps(args))) async with self.session.post(self.url, data=data) as resp: if resp.status == 200: r = await resp.json() From aa1fb9d5df4eda7ce0923775af1ef8ff79c67b97 Mon Sep 17 00:00:00 2001 From: SomberNight Date: Tue, 9 Jun 2020 17:55:16 +0200 Subject: [PATCH 4/4] win/mac binaries: rm jsonrpc* dependencies --- contrib/build-wine/deterministic.spec | 2 -- contrib/osx/osx.spec | 2 -- 2 files changed, 4 deletions(-) diff --git a/contrib/build-wine/deterministic.spec b/contrib/build-wine/deterministic.spec index 4509c956e..b08e77103 100644 --- a/contrib/build-wine/deterministic.spec +++ b/contrib/build-wine/deterministic.spec @@ -50,8 +50,6 @@ datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') datas += collect_data_files('bitbox02') -datas += collect_data_files('jsonrpcserver') -datas += collect_data_files('jsonrpcclient') # We don't put these files in to actually include them in the script but to make the Analysis method scan them for imports a = Analysis([home+'run_electrum', diff --git a/contrib/osx/osx.spec b/contrib/osx/osx.spec index 4a6a3a947..9e2cfd381 100644 --- a/contrib/osx/osx.spec +++ b/contrib/osx/osx.spec @@ -83,8 +83,6 @@ datas += collect_data_files('btchip') datas += collect_data_files('keepkeylib') datas += collect_data_files('ckcc') datas += collect_data_files('bitbox02') -datas += collect_data_files('jsonrpcserver') -datas += collect_data_files('jsonrpcclient') # Add the QR Scanner helper app datas += [(electrum + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app")]