diff --git a/dragonriders/.gitignore b/dragonriders/.gitignore index 340cf5a..c8ca8bf 100644 --- a/dragonriders/.gitignore +++ b/dragonriders/.gitignore @@ -1,7 +1,7 @@ *.swp assetchains komodotools_venv/ -docker-compose_assets_development.yml -docker-compose_assets_production.yml +docker-compose_assets_*.yml __pycache__ dokomodo.egg-info +assetchains_data.yaml diff --git a/dragonriders/dokomodo/cli.py b/dragonriders/dokomodo/cli.py index e4e031c..90ba66d 100755 --- a/dragonriders/dokomodo/cli.py +++ b/dragonriders/dokomodo/cli.py @@ -11,10 +11,14 @@ from py import path from sys import exit from os import path as expander from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException +import requests +import json +import re -# asset_data_url = ("https://raw.githubusercontent.com/patchkez/kmdplatform/" -# "master/yaml/data.yaml") +new_asset_data_url = ("https://raw.githubusercontent.com/patchkez/SuperNET/" + "separate_json_data_dev_cherrypick/iguana/coins/tmp_cleanup/assetchains_data.yml") +yamlname = 'assetchains_data.yaml' env = Environment(loader=FileSystemLoader('./dokomodo/templates/'), trim_blocks=True, lstrip_blocks=True) @@ -23,10 +27,80 @@ config_dir = path.local('/yaml_data') if config_dir.check() is False: config_dir = path.local('dokomodo/yaml') +global dump + + +# TEMPORARY CODE +# Temporary - download yaml file with iguana addcoin methods and supplies +def download_assets_data(): + global dump + file = requests.get(new_asset_data_url, stream=True) + # dump = file.raw + dump = file.text + + +def save_assets_data(): + global dump + newyaml = path.local(config_dir).join(yamlname) + + try: + myfile2 = newyaml.open(mode='w', ensure=True) + myfile2.write(dump) + except OSError as exception: + click.echo('File could not be opened in write mode or be written: {}'.format(exception)) + del dump + + +click.echo('Downloading preparsed {} as {}'.format(new_asset_data_url, yamlname)) +download_assets_data() +save_assets_data() + + +# This is common function for sending rpc requests to bitcoind and komodod +def send_request(assetchain_name, assetchain_rpcport, method): + assetchain_rpcuser = 'rpcuser' + assetchain_rpcpassword = 'rpcpassword' + + # request_url = ( + # 'http://' + asset_rpcuser + ':' + asset_rpcpassword + '@' + assetchain_name + ':' + + # assetchain_rpcport) + + rpc_connection = AuthServiceProxy("http://%s:%s@%s:%s" % (assetchain_rpcuser, + assetchain_rpcpassword, assetchain_name, int(assetchain_rpcport))) + + sendmethod = rpc_connection + '.' + method + try: + # rpc_connection.sendmany("", ctx.sendtomany_recipients) + sendmethod + except JSONRPCException as e: + click.echo("Error: %s" % e.error['message']) + + +# Common fucntion for sending any http API request e.g. to iguana +def post_rpc(url, payload, auth): + try: + r = requests.post(url, data=json.dumps(payload), auth=auth) + return(json.loads(r.text)) + except Exception as e: + print("Couldn't connect to " + url, e) + exit(0) + + +# Common functions for getting data from web servers +def get_rpc(url): + try: + r = requests.get(url) + # return(json.loads(r.text)) + return(r.text) + except Exception as e: + print("Couldn't connect to " + url, e) + exit(0) + class Config(object): def __init__(self, *args, **kwargs): self.config = config_dir.join('data.yaml') + self.new_config = config_dir.join('assetchains_data.yaml') self.config_ini = config_dir.join('config.ini') super(Config, self).__init__(*args, **kwargs) @@ -36,6 +110,7 @@ class Config(object): # Configure yaml loader yaml = YAML(typ='safe', pure=True) yaml.default_flow_style = True + yaml.width = 8096 # or some other big enough value to prevent line-wrap # Try to read yaml file try: self.config_data = yaml.load(self.config.read()) @@ -44,6 +119,21 @@ class Config(object): self.branches = self.config_data['assetchains'] self.seed_ip = gethostbyname(self.config_data['seed_host']) + def load_new_asset(self): + """Try to load the yaml""" + # Configure yaml loader + yaml = YAML(typ='safe', pure=True) + yaml.default_flow_style = True + # Try to read yaml file + try: + self.new_config_data = yaml.load(self.new_config.read()) + except OSError as exception: + click.echo('{} yaml file could not be read: {}'.format(self.config.read, exception)) + # self.branches = self.new_config_data['assetchains'] + # self.seed_ip = gethostbyname(self.config_data['seed_host']) + # self.supply = self + self.seed_ip2 = gethostbyname(self.new_config_data['seed_ip']) + def load_ini(self): # initialize INI parser ini_parser = ConfigParser() @@ -61,6 +151,11 @@ class Config(object): self.rpc_username = self.assetchains['rpc_username'] self.rpc_password = self.assetchains['rpc_password'] self.write_path_conf = self.assetchains['write_path_conf'] + self.iguana = ini_parser['IGUANA'] + self.production_coins = self.iguana['production_coins'] + self.development_coins = self.iguana['development_coins'] + self.iguana_url = self.iguana['iguana_url'] + self.iguana_home_dir = self.iguana['iguana_home_dir'] self.scaling_tests = ini_parser['SCALING_TESTING'] self.sendtomany_recipients = self.scaling_tests['sendtomany_recipients'] @@ -92,11 +187,12 @@ pass_config = click.make_pass_decorator(Config, ensure=True) @pass_config def cli(config): config.load() + config.load_new_asset() config.load_ini() @click.command('generate_docker_compose', - short_help='Generates docker-compose file with all assetchains') + short_help='OLD - Generates docker-compose file with all assetchains') @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production', 'test']), prompt=True) @@ -113,6 +209,42 @@ def generate_docker_compose(ctx, branch): ctx.write_config(dirname, filename=filename, templatized_config=templatized_config) +@click.command('generate_new_docker_compose', + short_help='PROD - Generates docker-compose file with all assetchains') +@click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production', + 'test']), prompt=True) +@click.option('-a', '--asset', required=False, help='name of assetchain in capital \ + letters e.g. SUPERNET') +@pass_config +def generate_new_docker_compose(ctx, branch, asset): + """ TODO """ + filename = 'docker-compose_assets_' + branch + '.yml' + dirname = "./" + click.echo('Writing new docker compose file into: {}'.format(filename)) + + yaml = YAML(typ='safe', pure=True) + yaml.default_flow_style = True + dic = {} + + def filtered_yaml(): + coins = branch + '_coins_assets' + for assetchain_key in ctx.assetchains[coins].split(', '): + x = ctx.new_config_data['assetchains'][assetchain_key] + dic[assetchain_key] = x + if asset and asset == assetchain_key: + pass + elif asset: + pass + else: + pass + return dic + + template = env.get_template('docker-compose-new-template.conf.j2') + templatized_config = template.render(items=filtered_yaml(), + seed_ip=ctx.seed_ip2, mined=ctx.mined_coins, btcpubkey=ctx.btcpubkey) + ctx.write_config(dirname, filename=filename, templatized_config=templatized_config) + + @click.command('assetchains', short_help='Replacement for assetchains script') @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production', 'test']), @@ -151,7 +283,9 @@ def generate_assetchains_conf(ctx, branch, asset): asset_templatized_config = asset_template.render( rpcuser=ctx.rpc_username, rpcpassword=ctx.rpc_password, - rpcport=ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port'], + # rpcport=ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port'], + rpcport=ctx.new_config_data['assetchains'][assetchain_name]['iguana_payload'][ + 'rpc'], btcpubkey=ctx.btcpubkey ) @@ -160,63 +294,125 @@ def generate_assetchains_conf(ctx, branch, asset): # return click.echo(asset_templatized_config) return asset_templatized_config - for assetchain_name in ctx.config_data['assetchains'][branch]: - if asset and asset == assetchain_name: - click.echo('Writing CONFIG_FILE: {}'.format(assetchain_name)) - templatize(assetchain_name) + coins = branch + '_coins_assets' + for assetchain_key in ctx.assetchains[coins].split(', '): + # for assetchain_name in ctx.config_data['assetchains'][branch]: + if asset and asset == assetchain_key: + click.echo('Writing CONFIG_FILE: {}'.format(assetchain_key)) + templatize(assetchain_key) elif asset: pass else: - click.echo('Writing CONFIG_FILE: {}'.format(assetchain_name)) - templatize(assetchain_name) + click.echo('Writing CONFIG_FILE: {}'.format(assetchain_key)) + templatize(assetchain_key) -@click.command('sendmany_assetchains', short_help='Import private key into assetchains') +@click.command('sendmany_assetchains', short_help='WIP - Sendmany to one or many assetchains') @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production', 'test'])) @click.option('-a', '--asset', required=False) @pass_config def sendmany_assetchains(ctx, branch, asset): - def send_request(assetchain_name, assetchain_rpcport): - assetchain_rpcuser = 'rpcuser' - assetchain_rpcpassword = 'rpcpassword' - - # request_url = ( - # 'http://' + asset_rpcuser + ':' + asset_rpcpassword + '@' + assetchain_name + ':' + - # assetchain_rpcport) - - rpc_connection = AuthServiceProxy("http://%s:%s@%s:%s" % (assetchain_rpcuser, - assetchain_rpcpassword, assetchain_name, int(assetchain_rpcport))) - - try: - rpc_connection.sendmany("", ctx.sendtomany_recipients) - except JSONRPCException as e: - click.echo("Error: %s" % e.error['message']) - counter = 0 while counter < float(ctx.number_of_requests): # click.echo(ctx.config_data['assetchains'][branch]) for assetchain_name in ctx.config_data['assetchains'][branch]: + method = 'sendmany'("", ctx.sendtomany_recipients) # click.echo(type(assetchain_name)) rpc_port = ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port'] if asset and asset == assetchain_name: click.echo('Sending request to: {}'.format(assetchain_name)) - send_request(assetchain_name, rpc_port) + send_request(assetchain_name, rpc_port, method) elif asset: pass else: click.echo('Sending request to: {}'.format(assetchain_name)) - send_request(assetchain_name, rpc_port) + send_request(assetchain_name, rpc_port, method) counter += 1 sleep(ctx.delay_between_requests) +@click.command('start_iguana', short_help='Add all methods into iguana') +@click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production'])) +@click.option('-a', '--asset', required=False, help='name of assetchain in capital \ + letters e.g. SUPERNET') +@click.option('-p', '--password', prompt=True, hide_input=True, + confirmation_prompt=True, help='iguana passphrase') +@pass_config +def start_iguana(ctx, branch, asset, password): + url = ctx.iguana_url + # No authentication is needed for iguana + auth = ('', '') + + # My IP + # get public IP address of this host + myip = 'http://' + ctx.new_config_data['check_my_ip'] + response = get_rpc(myip).rstrip() + click.echo('My IP address is: {}'.format(response)) + myipmethod = ctx.new_config_data['misc_methods']['supernet_myip'] + myipmethod['ipaddr'] = response + # click.echo('MY IP method: {}'.format(myipmethod)) + post_rpc(url, myipmethod, auth) + + # Add notaries + for notary in ctx.new_config_data['misc_methods']['notaries']: + click.echo('Adding notary: {}'.format(notary)) + post_rpc(url, ctx.new_config_data['misc_methods']['notaries'][notary], auth) + + # Walletpassphrase + click.echo('Adding walletpassphrase!') + wallet = ctx.new_config_data['misc_methods']['wallet_passphrase'] + # Replace password with the one provided by user + wallet['params'][0] = password + # click.echo(wallet) + post_rpc(url, wallet, auth) + + # Add coins + DPOW + coins = branch + '_coins' + for assetchain_key in ctx.iguana[coins].split(', '): + # Read only assetchains payloads + payload = ctx.new_config_data['assetchains'][assetchain_key]['iguana_payload'] + + # Replace ${HOME#/} with value in our INI file + # remove first '/' + home = re.sub(r"\/", "", ctx.iguana_home_dir, 1) + # Read value of 'path' key + line = payload['path'] + # Substitute + newline = re.sub(r"\$\{HOME\#\/\}", home, line) + # Update value in loaded dictionary + ctx.new_config_data['assetchains'][assetchain_key]['iguana_payload']['path'] = newline + + # Dpow + dpow = ctx.new_config_data['misc_methods']['dpow'] + dpow['pubkey'] = ctx.btcpubkey + dpow['symbol'] = assetchain_key + # click.echo(wallet) + if asset and asset == assetchain_key: + click.echo('Sending addcoin request to: {}'.format(assetchain_key)) + post_rpc(url, payload, auth) + sleep(3) + click.echo('Sending dpow {} request to: {}'.format(dpow, assetchain_key)) + post_rpc(url, dpow, auth) + elif asset: + pass + else: + click.echo('Sending addcoin request to: {}'.format(assetchain_key)) + post_rpc(url, payload, auth) + sleep(10) + click.echo('Sending dpow {} request to: {}'.format(dpow, assetchain_key)) + post_rpc(url, dpow, auth) + sleep(10) + + # Add functions into cli() function which is main group for all commands cli.add_command(generate_docker_compose) +cli.add_command(generate_new_docker_compose) cli.add_command(assetchains) cli.add_command(generate_assetchains_conf) cli.add_command(sendmany_assetchains) +cli.add_command(start_iguana) if __name__ == "__main__": diff --git a/dragonriders/dokomodo/templates/docker-compose-new-template.conf.j2 b/dragonriders/dokomodo/templates/docker-compose-new-template.conf.j2 new file mode 100644 index 0000000..8bc2694 --- /dev/null +++ b/dragonriders/dokomodo/templates/docker-compose-new-template.conf.j2 @@ -0,0 +1,57 @@ +# vim: ts=2 sw=2 et +version: '3' + +services: + {% for assetname, asset in items.items() %} + {{ assetname }}: + image: kmdplatform_komodo + # network_mode: "host" + networks: + - komodo + ports: + {% if asset.iguana_payload.rpc %} + - "127.0.0.1:{{ asset.iguana_payload.rpc }}:{{ asset.iguana_payload.rpc }}" + {% endif %} + {% if asset.iguana_payload.p2p %} + - "{{ asset.iguana_payload.p2p }}:{{ asset.iguana_payload.p2p }}" + {% endif %} + volumes: + - komodo-data:/home/komodo/.komodo + - shared-data:/home/komodo/.shared:ro + environment: + - ASSET_RPC_USER=${RPC_USER} + - ASSET_RPC_PASSWORD=${RPC_PASSWORD} + - ASSET_RPC_WORKQUEUE=64 + - ASSET_BIND=127.0.0.1 + - ASSET_RPC_BIND=0.0.0.0 + - ASSET_RPC_ALLOWIP=0.0.0.0/0 + - ASSET_RPC_PORT={{ asset.iguana_payload.p2p }} + - ASSET_NAME={{ assetname }} + command: > + bash -c "komodod -pubkey={{ btcpubkey }} + -ac_name={{ assetname }} + -ac_supply={{ asset.supply }} + -addnode={{ seed_ip }}{% for mine in mined %} + {% if assetname in mine %} -gen{% endif %} + {% if 'random' in mine %} + {% if range(0, 32767) | random % 10 == 1 %} + -gen{% endif %} + {% endif %} + {% endfor %}" + + {% endfor %} + +volumes: + komodo-data: + driver_opts: + type: none + device: ${KOMODO_DATA} + o: bind + shared-data: + driver_opts: + type: none + device: ${SHARED_DATA} + o: bind + +networks: + komodo: diff --git a/dragonriders/dokomodo/templates/docker-compose-template.conf.j2 b/dragonriders/dokomodo/templates/docker-compose-template.conf.j2 index 25603a1..425a595 100644 --- a/dragonriders/dokomodo/templates/docker-compose-template.conf.j2 +++ b/dragonriders/dokomodo/templates/docker-compose-template.conf.j2 @@ -5,7 +5,9 @@ services: {% for assetname, asset in items.items() %} {{ assetname }}: image: kmdplatform_komodo - network_mode: "host" + # network_mode: "host" + networks: + - komodo ports: {% if asset.rpc_port %} - "127.0.0.1:{{ asset.rpc_port }}:{{ asset.rpc_port }}" @@ -21,22 +23,22 @@ services: - ASSET_RPC_PASSWORD=${RPC_PASSWORD} - ASSET_RPC_WORKQUEUE=64 - ASSET_BIND=127.0.0.1 - - ASSET_RPC_BIND=127.0.0.1 - - ASSET_RPC_ALLOWIP=0.0.0.0 + - ASSET_RPC_BIND=0.0.0.0 + - ASSET_RPC_ALLOWIP=0.0.0.0/0 - ASSET_RPC_PORT={{ asset.p2p_port }} - ASSET_NAME={{ assetname }} - command: > + command: > bash -c "komodod -pubkey={{ btcpubkey }} - -ac_name={{ assetname }} - -ac_supply={{ asset.amount }} + -ac_name={{ assetname }} + -ac_supply={{ asset.amount }} -addnode={{ seed_ip }}{% for mine in mined %} {% if assetname in mine %} -gen{% endif %} {% if 'random' in mine %} - {% if range(0, 32767) | random % 10 == 1 %} + {% if range(0, 32767) | random % 10 == 1 %} -gen{% endif %} {% endif %} {% endfor %}" - + {% endfor %} volumes: @@ -50,3 +52,6 @@ volumes: type: none device: ${SHARED_DATA} o: bind + +networks: + komodo: diff --git a/dragonriders/dokomodo/yaml/config.ini b/dragonriders/dokomodo/yaml/config.ini index 342cacd..5020038 100644 --- a/dragonriders/dokomodo/yaml/config.ini +++ b/dragonriders/dokomodo/yaml/config.ini @@ -22,9 +22,32 @@ delay_asset = 20 rpc_username = rpcusernameINI rpc_password = rpcpasswordINI +# this is temporary, we need to find aout way from where to read IPs - etcd? +rpc_ip = 127.0.0.1 write_path_conf = ~/.komodo +# List of assetchains to be started, it does not mean when they ar estarted that they are also notarized!!! +# List is used to create configuration files for assetchains +# to access its data in central registry file +production_coins_assets = REVS, SUPERNET, DEX, PANGEA, JUMBLR, BET, CRYPTO, HODL, MSHARK, BOTS, MGW, COQUI, WLC, KV, CEAL, MESH, MNZ, AXO, ETOMIC, BTCH, VOTE2018, PIZZA, BEER, NINJA, NINJA, OOT, BNTN, CHAIN, PRLPAY +development_coins_assets = PIZZA, BEER + + +[IGUANA] + +iguana_home_dir = /home/iguana + +iguana_host = localhost +iguana_rpc_port = 7776 +# extra 's' is due to how configparser interpolates variables +iguana_url = http://%(iguana_host)s:%(iguana_rpc_port)s + +# List of chains which will be notarized!!! +# This will be used mostly for iguana methods, this list will be used in loop and key of coin will be used +# to access its data in central registry file +production_coins = BTC, LTC, KMD, CHIPS, REVS, SUPERNET, DEX, PANGEA, JUMBLR, BET, CRYPTO, HODL, MSHARK, BOTS, MGW, COQUI, WLC, KV, CEAL, MESH, MNZ, AXO, ETOMIC, BTCH, VOTE2018, NINJA, OOT, BNTN, CHAIN, PRLPAY +development_coins = PIZZA, BEER [SCALING_TESTING]