Merge pull request #20 from patchkez/requirements_requests

New central registry file + start_iguana subcommand

dragonriders/.gitignore

@@ -1,7 +1,7 @@
 *.swp
 assetchains
 komodotools_venv/
-docker-compose_assets_development.yml
-docker-compose_assets_production.yml
+docker-compose_assets_*.yml
 __pycache__
 dokomodo.egg-info
+assetchains_data.yaml

dragonriders/dokomodo/cli.py

@@ -11,10 +11,14 @@ from py import path
 from sys import exit
 from os import path as expander
 from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+import requests
+import json
+import re
 
 
-# asset_data_url = ("https://raw.githubusercontent.com/patchkez/kmdplatform/"
-#    "master/yaml/data.yaml")
+new_asset_data_url = ("https://raw.githubusercontent.com/patchkez/SuperNET/"
+    "separate_json_data_dev_cherrypick/iguana/coins/tmp_cleanup/assetchains_data.yml")
+yamlname = 'assetchains_data.yaml'
 
 env = Environment(loader=FileSystemLoader('./dokomodo/templates/'), trim_blocks=True,
     lstrip_blocks=True)
@@ -23,10 +27,80 @@ config_dir = path.local('/yaml_data')
 if config_dir.check() is False:
     config_dir = path.local('dokomodo/yaml')
 
+global dump
+
+
+# TEMPORARY CODE
+# Temporary - download yaml file with iguana addcoin methods and supplies
+def download_assets_data():
+    global dump
+    file = requests.get(new_asset_data_url, stream=True)
+    # dump = file.raw
+    dump = file.text
+
+
+def save_assets_data():
+    global dump
+    newyaml = path.local(config_dir).join(yamlname)
+
+    try:
+        myfile2 = newyaml.open(mode='w', ensure=True)
+        myfile2.write(dump)
+    except OSError as exception:
+        click.echo('File could not be opened in write mode or be written: {}'.format(exception))
+    del dump
+
+
+click.echo('Downloading preparsed {} as {}'.format(new_asset_data_url, yamlname))
+download_assets_data()
+save_assets_data()
+
+
+# This is common function for sending rpc requests to bitcoind and komodod
+def send_request(assetchain_name, assetchain_rpcport, method):
+    assetchain_rpcuser = 'rpcuser'
+    assetchain_rpcpassword = 'rpcpassword'
+
+    # request_url = (
+    #     'http://' + asset_rpcuser + ':' + asset_rpcpassword + '@' + assetchain_name + ':' +
+    #     assetchain_rpcport)
+
+    rpc_connection = AuthServiceProxy("http://%s:%s@%s:%s" % (assetchain_rpcuser,
+        assetchain_rpcpassword, assetchain_name, int(assetchain_rpcport)))
+
+    sendmethod = rpc_connection + '.' + method
+    try:
+        # rpc_connection.sendmany("", ctx.sendtomany_recipients)
+        sendmethod
+    except JSONRPCException as e:
+        click.echo("Error: %s" % e.error['message'])
+
+
+# Common fucntion for sending any http API request e.g. to iguana
+def post_rpc(url, payload, auth):
+    try:
+        r = requests.post(url, data=json.dumps(payload), auth=auth)
+        return(json.loads(r.text))
+    except Exception as e:
+        print("Couldn't connect to " + url, e)
+    exit(0)
+
+
+# Common functions for getting data from web servers
+def get_rpc(url):
+    try:
+        r = requests.get(url)
+        # return(json.loads(r.text))
+        return(r.text)
+    except Exception as e:
+        print("Couldn't connect to " + url, e)
+    exit(0)
+
 
 class Config(object):
     def __init__(self, *args, **kwargs):
         self.config = config_dir.join('data.yaml')
+        self.new_config = config_dir.join('assetchains_data.yaml')
         self.config_ini = config_dir.join('config.ini')
 
         super(Config, self).__init__(*args, **kwargs)
@@ -36,6 +110,7 @@ class Config(object):
         # Configure yaml loader
         yaml = YAML(typ='safe', pure=True)
         yaml.default_flow_style = True
+        yaml.width = 8096  # or some other big enough value to prevent line-wrap
         # Try to read yaml file
         try:
             self.config_data = yaml.load(self.config.read())
@@ -44,6 +119,21 @@ class Config(object):
         self.branches = self.config_data['assetchains']
         self.seed_ip = gethostbyname(self.config_data['seed_host'])
 
+    def load_new_asset(self):
+        """Try to load the yaml"""
+        # Configure yaml loader
+        yaml = YAML(typ='safe', pure=True)
+        yaml.default_flow_style = True
+        # Try to read yaml file
+        try:
+            self.new_config_data = yaml.load(self.new_config.read())
+        except OSError as exception:
+            click.echo('{} yaml file could not be read: {}'.format(self.config.read, exception))
+        # self.branches = self.new_config_data['assetchains']
+        # self.seed_ip = gethostbyname(self.config_data['seed_host'])
+        # self.supply = self
+        self.seed_ip2 = gethostbyname(self.new_config_data['seed_ip'])
+
     def load_ini(self):
         # initialize INI parser
         ini_parser = ConfigParser()
@@ -61,6 +151,11 @@ class Config(object):
         self.rpc_username = self.assetchains['rpc_username']
         self.rpc_password = self.assetchains['rpc_password']
         self.write_path_conf = self.assetchains['write_path_conf']
+        self.iguana = ini_parser['IGUANA']
+        self.production_coins = self.iguana['production_coins']
+        self.development_coins = self.iguana['development_coins']
+        self.iguana_url = self.iguana['iguana_url']
+        self.iguana_home_dir = self.iguana['iguana_home_dir']
 
         self.scaling_tests = ini_parser['SCALING_TESTING']
         self.sendtomany_recipients = self.scaling_tests['sendtomany_recipients']
@@ -92,11 +187,12 @@ pass_config = click.make_pass_decorator(Config, ensure=True)
 @pass_config
 def cli(config):
     config.load()
+    config.load_new_asset()
     config.load_ini()
 
 
 @click.command('generate_docker_compose',
-    short_help='Generates docker-compose file with all assetchains')
+    short_help='OLD - Generates docker-compose file with all assetchains')
 @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production',
     'test']),
     prompt=True)
@@ -113,6 +209,42 @@ def generate_docker_compose(ctx, branch):
     ctx.write_config(dirname, filename=filename, templatized_config=templatized_config)
 
 
+@click.command('generate_new_docker_compose',
+    short_help='PROD - Generates docker-compose file with all assetchains')
+@click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production',
+    'test']), prompt=True)
+@click.option('-a', '--asset', required=False, help='name of assetchain in capital \
+    letters e.g. SUPERNET')
+@pass_config
+def generate_new_docker_compose(ctx, branch, asset):
+    """ TODO """
+    filename = 'docker-compose_assets_' + branch + '.yml'
+    dirname = "./"
+    click.echo('Writing new docker compose file into: {}'.format(filename))
+
+    yaml = YAML(typ='safe', pure=True)
+    yaml.default_flow_style = True
+    dic = {}
+
+    def filtered_yaml():
+        coins = branch + '_coins_assets'
+        for assetchain_key in ctx.assetchains[coins].split(', '):
+            x = ctx.new_config_data['assetchains'][assetchain_key]
+            dic[assetchain_key] = x
+            if asset and asset == assetchain_key:
+                pass
+            elif asset:
+                pass
+            else:
+                pass
+        return dic
+
+    template = env.get_template('docker-compose-new-template.conf.j2')
+    templatized_config = template.render(items=filtered_yaml(),
+        seed_ip=ctx.seed_ip2, mined=ctx.mined_coins, btcpubkey=ctx.btcpubkey)
+    ctx.write_config(dirname, filename=filename, templatized_config=templatized_config)
+
+
 @click.command('assetchains', short_help='Replacement for assetchains script')
 @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production',
     'test']),
@@ -151,7 +283,9 @@ def generate_assetchains_conf(ctx, branch, asset):
         asset_templatized_config = asset_template.render(
             rpcuser=ctx.rpc_username,
             rpcpassword=ctx.rpc_password,
-            rpcport=ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port'],
+            # rpcport=ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port'],
+            rpcport=ctx.new_config_data['assetchains'][assetchain_name]['iguana_payload'][
+                'rpc'],
             btcpubkey=ctx.btcpubkey
         )
 
@@ -160,63 +294,125 @@ def generate_assetchains_conf(ctx, branch, asset):
         # return click.echo(asset_templatized_config)
         return asset_templatized_config
 
-    for assetchain_name in ctx.config_data['assetchains'][branch]:
-        if asset and asset == assetchain_name:
-            click.echo('Writing CONFIG_FILE: {}'.format(assetchain_name))
-            templatize(assetchain_name)
+    coins = branch + '_coins_assets'
+    for assetchain_key in ctx.assetchains[coins].split(', '):
+        # for assetchain_name in ctx.config_data['assetchains'][branch]:
+        if asset and asset == assetchain_key:
+            click.echo('Writing CONFIG_FILE: {}'.format(assetchain_key))
+            templatize(assetchain_key)
         elif asset:
             pass
         else:
-            click.echo('Writing CONFIG_FILE: {}'.format(assetchain_name))
-            templatize(assetchain_name)
+            click.echo('Writing CONFIG_FILE: {}'.format(assetchain_key))
+            templatize(assetchain_key)
 
 
-@click.command('sendmany_assetchains', short_help='Import private key into assetchains')
+@click.command('sendmany_assetchains', short_help='WIP - Sendmany to one or many assetchains')
 @click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production',
     'test']))
 @click.option('-a', '--asset', required=False)
 @pass_config
 def sendmany_assetchains(ctx, branch, asset):
 
-    def send_request(assetchain_name, assetchain_rpcport):
-        assetchain_rpcuser = 'rpcuser'
-        assetchain_rpcpassword = 'rpcpassword'
-
-        # request_url = (
-        #     'http://' + asset_rpcuser + ':' + asset_rpcpassword + '@' + assetchain_name + ':' +
-        #     assetchain_rpcport)
-
-        rpc_connection = AuthServiceProxy("http://%s:%s@%s:%s" % (assetchain_rpcuser,
-            assetchain_rpcpassword, assetchain_name, int(assetchain_rpcport)))
-
-        try:
-            rpc_connection.sendmany("", ctx.sendtomany_recipients)
-        except JSONRPCException as e:
-            click.echo("Error: %s" % e.error['message'])
-
     counter = 0
     while counter < float(ctx.number_of_requests):
         # click.echo(ctx.config_data['assetchains'][branch])
         for assetchain_name in ctx.config_data['assetchains'][branch]:
+            method = 'sendmany'("", ctx.sendtomany_recipients)
             # click.echo(type(assetchain_name))
             rpc_port = ctx.config_data['assetchains'][branch][assetchain_name]['rpc_port']
             if asset and asset == assetchain_name:
                 click.echo('Sending request to: {}'.format(assetchain_name))
-                send_request(assetchain_name, rpc_port)
+                send_request(assetchain_name, rpc_port, method)
             elif asset:
                 pass
             else:
                 click.echo('Sending request to: {}'.format(assetchain_name))
-                send_request(assetchain_name, rpc_port)
+                send_request(assetchain_name, rpc_port, method)
         counter += 1
         sleep(ctx.delay_between_requests)
 
 
+@click.command('start_iguana', short_help='Add all methods into iguana')
+@click.option('-b', '--branch', required=True, type=click.Choice(['development', 'production']))
+@click.option('-a', '--asset', required=False, help='name of assetchain in capital \
+    letters e.g. SUPERNET')
+@click.option('-p', '--password', prompt=True, hide_input=True,
+    confirmation_prompt=True, help='iguana passphrase')
+@pass_config
+def start_iguana(ctx, branch, asset, password):
+    url = ctx.iguana_url
+    # No authentication is needed for iguana
+    auth = ('', '')
+
+    # My IP
+    # get public IP address of this host
+    myip = 'http://' + ctx.new_config_data['check_my_ip']
+    response = get_rpc(myip).rstrip()
+    click.echo('My IP address is: {}'.format(response))
+    myipmethod = ctx.new_config_data['misc_methods']['supernet_myip']
+    myipmethod['ipaddr'] = response
+    # click.echo('MY IP method: {}'.format(myipmethod))
+    post_rpc(url, myipmethod, auth)
+
+    #  Add notaries
+    for notary in ctx.new_config_data['misc_methods']['notaries']:
+        click.echo('Adding notary: {}'.format(notary))
+        post_rpc(url, ctx.new_config_data['misc_methods']['notaries'][notary], auth)
+
+    # Walletpassphrase
+    click.echo('Adding walletpassphrase!')
+    wallet = ctx.new_config_data['misc_methods']['wallet_passphrase']
+    # Replace password with the one provided by user
+    wallet['params'][0] = password
+    # click.echo(wallet)
+    post_rpc(url, wallet, auth)
+
+    # Add coins + DPOW
+    coins = branch + '_coins'
+    for assetchain_key in ctx.iguana[coins].split(', '):
+        # Read only assetchains payloads
+        payload = ctx.new_config_data['assetchains'][assetchain_key]['iguana_payload']
+
+        # Replace ${HOME#/} with value in our INI file
+        # remove first '/'
+        home = re.sub(r"\/", "", ctx.iguana_home_dir, 1)
+        # Read value of 'path' key
+        line = payload['path']
+        # Substitute
+        newline = re.sub(r"\$\{HOME\#\/\}", home, line)
+        # Update value in loaded dictionary
+        ctx.new_config_data['assetchains'][assetchain_key]['iguana_payload']['path'] = newline
+
+        # Dpow
+        dpow = ctx.new_config_data['misc_methods']['dpow']
+        dpow['pubkey'] = ctx.btcpubkey
+        dpow['symbol'] = assetchain_key
+    # click.echo(wallet)
+        if asset and asset == assetchain_key:
+            click.echo('Sending addcoin request to: {}'.format(assetchain_key))
+            post_rpc(url, payload, auth)
+            sleep(3)
+            click.echo('Sending dpow {} request to: {}'.format(dpow, assetchain_key))
+            post_rpc(url, dpow, auth)
+        elif asset:
+            pass
+        else:
+            click.echo('Sending addcoin request to: {}'.format(assetchain_key))
+            post_rpc(url, payload, auth)
+            sleep(10)
+            click.echo('Sending dpow {} request to: {}'.format(dpow, assetchain_key))
+            post_rpc(url, dpow, auth)
+            sleep(10)
+
+
 # Add functions into cli() function which is main group for all commands
 cli.add_command(generate_docker_compose)
+cli.add_command(generate_new_docker_compose)
 cli.add_command(assetchains)
 cli.add_command(generate_assetchains_conf)
 cli.add_command(sendmany_assetchains)
+cli.add_command(start_iguana)
 
 
 if __name__ == "__main__":

dragonriders/dokomodo/templates/docker-compose-new-template.conf.j2

@@ -0,0 +1,57 @@
+# vim: ts=2 sw=2 et
+version: '3'
+
+services:
+  {% for assetname, asset in items.items() %}
+  {{ assetname }}:
+    image: kmdplatform_komodo
+    # network_mode: "host"
+    networks:
+      - komodo
+    ports:
+      {% if asset.iguana_payload.rpc %}
+      - "127.0.0.1:{{ asset.iguana_payload.rpc }}:{{ asset.iguana_payload.rpc }}"
+      {% endif %}
+      {% if asset.iguana_payload.p2p %}
+      - "{{ asset.iguana_payload.p2p }}:{{ asset.iguana_payload.p2p }}"
+      {% endif %}
+    volumes:
+      - komodo-data:/home/komodo/.komodo
+      - shared-data:/home/komodo/.shared:ro
+    environment:
+      - ASSET_RPC_USER=${RPC_USER}
+      - ASSET_RPC_PASSWORD=${RPC_PASSWORD}
+      - ASSET_RPC_WORKQUEUE=64
+      - ASSET_BIND=127.0.0.1
+      - ASSET_RPC_BIND=0.0.0.0
+      - ASSET_RPC_ALLOWIP=0.0.0.0/0
+      - ASSET_RPC_PORT={{ asset.iguana_payload.p2p }}
+      - ASSET_NAME={{ assetname }}
+    command: >
+      bash -c "komodod -pubkey={{ btcpubkey }}
+      -ac_name={{ assetname }}
+      -ac_supply={{ asset.supply }}
+      -addnode={{ seed_ip }}{% for mine in mined %}
+      {% if assetname in mine %} -gen{% endif %}
+      {% if 'random' in mine %}
+      {% if range(0, 32767) | random % 10 == 1 %}
+      -gen{% endif %}
+      {% endif %}
+      {% endfor %}"
+
+  {% endfor %}
+
+volumes:
+  komodo-data:
+    driver_opts:
+      type: none
+      device: ${KOMODO_DATA}
+      o: bind
+  shared-data:
+    driver_opts:
+      type: none
+      device: ${SHARED_DATA}
+      o: bind
+
+networks:
+  komodo:

dragonriders/dokomodo/templates/docker-compose-template.conf.j2

@@ -5,7 +5,9 @@ services:
   {% for assetname, asset in items.items() %}
   {{ assetname }}:
     image: kmdplatform_komodo
-    network_mode: "host" 
+    # network_mode: "host"
+    networks:
+      - komodo
     ports:
       {% if asset.rpc_port %}
       - "127.0.0.1:{{ asset.rpc_port }}:{{ asset.rpc_port }}"
@@ -21,22 +23,22 @@ services:
       - ASSET_RPC_PASSWORD=${RPC_PASSWORD}
       - ASSET_RPC_WORKQUEUE=64
       - ASSET_BIND=127.0.0.1
-      - ASSET_RPC_BIND=127.0.0.1
-      - ASSET_RPC_ALLOWIP=0.0.0.0
+      - ASSET_RPC_BIND=0.0.0.0
+      - ASSET_RPC_ALLOWIP=0.0.0.0/0
       - ASSET_RPC_PORT={{ asset.p2p_port }}
       - ASSET_NAME={{ assetname }}
-    command: > 
+    command: >
       bash -c "komodod -pubkey={{ btcpubkey }}
-      -ac_name={{ assetname }} 
-      -ac_supply={{ asset.amount }} 
+      -ac_name={{ assetname }}
+      -ac_supply={{ asset.amount }}
       -addnode={{ seed_ip }}{% for mine in mined %}
       {% if assetname in mine %} -gen{% endif %}
       {% if 'random' in mine %}
-      {% if range(0, 32767) | random % 10 == 1 %} 
+      {% if range(0, 32767) | random % 10 == 1 %}
       -gen{% endif %}
       {% endif %}
       {% endfor %}"
-  
+
   {% endfor %}
 
 volumes:
@@ -50,3 +52,6 @@ volumes:
       type: none
       device: ${SHARED_DATA}
       o: bind
+
+networks:
+  komodo:

dragonriders/dokomodo/yaml/config.ini

@@ -22,9 +22,32 @@ delay_asset = 20
 
 rpc_username = rpcusernameINI
 rpc_password = rpcpasswordINI
+# this is temporary, we need to find aout way from where to read IPs - etcd?
+rpc_ip = 127.0.0.1
 
 write_path_conf = ~/.komodo
 
+# List of assetchains to be started, it does not mean when they ar estarted that they are also notarized!!!
+# List is used to create configuration files for assetchains
+# to access its data in central registry file
+production_coins_assets = REVS, SUPERNET, DEX, PANGEA, JUMBLR, BET, CRYPTO, HODL, MSHARK, BOTS, MGW, COQUI, WLC, KV, CEAL, MESH, MNZ, AXO, ETOMIC, BTCH, VOTE2018, PIZZA, BEER, NINJA, NINJA, OOT, BNTN, CHAIN, PRLPAY
+development_coins_assets = PIZZA, BEER
+
+
+[IGUANA]
+
+iguana_home_dir = /home/iguana
+
+iguana_host = localhost
+iguana_rpc_port = 7776
+# extra 's' is due to how configparser interpolates variables
+iguana_url = http://%(iguana_host)s:%(iguana_rpc_port)s
+
+# List of chains which will be notarized!!!
+# This will be used mostly for iguana methods, this list will be used in loop and key of coin will be used
+# to access its data in central registry file
+production_coins = BTC, LTC, KMD, CHIPS, REVS, SUPERNET, DEX, PANGEA, JUMBLR, BET, CRYPTO, HODL, MSHARK, BOTS, MGW, COQUI, WLC, KV, CEAL, MESH, MNZ, AXO, ETOMIC, BTCH, VOTE2018, NINJA, OOT, BNTN, CHAIN, PRLPAY
+development_coins = PIZZA, BEER
 
 
 [SCALING_TESTING]