Merge pull request #405 from jl777/spvdex

Spvdex

.gitignore

@@ -487,3 +487,5 @@ iguana/confs/97f18454bb61e9eb7a827cfbefe42fbf7ae2832dc74c4812bdaef8bcf5c10474
 iguana/DB/PRICES/.tmpmarker
 
 iguana/DB/KMD/0/.tmpmarker
+
+*.swp

iguana/exchanges/LP_RTmetrics.c

@@ -0,0 +1,65 @@
+
+/******************************************************************************
+ * Copyright © 2014-2017 The SuperNET Developers.                             *
+ *                                                                            *
+ * See the AUTHORS, DEVELOPER-AGREEMENT and LICENSE files at                  *
+ * the top-level directory of this distribution for the individual copyright  *
+ * holder information and the developer policies on copyright and licensing.  *
+ *                                                                            *
+ * Unless otherwise agreed in a custom licensing agreement, no part of the    *
+ * SuperNET software, including this file may be copied, modified, propagated *
+ * or distributed except according to the terms contained in the LICENSE file *
+ *                                                                            *
+ * Removal or modification of this copyright notice is prohibited.            *
+ *                                                                            *
+ ******************************************************************************/
+//
+//  LP_RTmetrics.c
+//  marketmaker
+//
+
+cJSON *LP_RTmetrics_sort(cJSON *rawasks,int32_t numasks,double maxprice,double relvolume)
+{
+    cJSON *array,*item,*statsjson,*swaps=0; int32_t i,numswaps=0; bits256 zero; uint32_t futuretime; double price; char *retstr;
+    futuretime = (uint32_t)time(NULL) + 3600*100;
+    memset(zero.bytes,0,sizeof(zero));
+    if ( (retstr= LP_statslog_disp(100,futuretime,futuretime,"",zero)) != 0 )
+    {
+        if ( (statsjson= cJSON_Parse(retstr)) != 0 )
+        {
+            if ( (swaps= jarray(&numswaps,statsjson,"swaps")) != 0 )
+            {
+                if ( numswaps > 0 )
+                    swaps = jduplicate(swaps);
+                else swaps = 0;
+            }
+            free_json(statsjson);
+        }
+        free(retstr);
+    }
+    //if ( numswaps == 0 || swaps == 0 )
+        return(0);
+    printf("calc RTmetrics for (%s)\n",jprint(swaps,0));
+    /*jadd64bits(item,"aliceid",sp->aliceid);
+    jaddbits256(item,"src",sp->Q.srchash);
+    jaddstr(item,"base",sp->Q.srccoin);
+    jaddnum(item,"basevol",dstr(sp->Q.satoshis));
+    jaddbits256(item,"dest",sp->Q.desthash);
+    jaddstr(item,"rel",sp->Q.destcoin);
+    jaddnum(item,"relvol",dstr(sp->Q.destsatoshis));
+    jaddnum(item,"price",sp->qprice);
+    jaddnum(item,"requestid",sp->Q.R.requestid);
+    jaddnum(item,"quoteid",sp->Q.R.quoteid);
+    */
+    array = cJSON_CreateArray();
+    for (i=0; i<numasks; i++)
+    {
+        item = jitem(rawasks,i);
+        price = jdouble(item,"price");
+        if ( price > maxprice )
+            break;
+        jaddi(array,jduplicate(item));
+    }
+    free_json(swaps);
+    return(array);
+}

iguana/exchanges/LP_include.h

@@ -36,7 +36,8 @@ void emscripten_usleep(int32_t x);
 
 #define LP_HTTP_TIMEOUT 3 // 1 is too small due to edge cases of time(NULL)
 #define LP_AUTOTRADE_TIMEOUT 10
-#define ELECTRUM_TIMEOUT 15
+#define ELECTRUM_TIMEOUT 10
+#define LP_ELECTRUM_MAXERRORS 3
 #define LP_MEMPOOL_TIMEINCR 10
 
 #define LP_COMMAND_SENDSOCK NN_PUSH

iguana/exchanges/LP_nativeDEX.c

@@ -18,6 +18,8 @@
 //  LP_nativeDEX.c
 //  marketmaker
 //
+//MERKLE DIDNT VERIFY.ZEC 96d8484efb1f96e2a692f572d2b3bbc49a59bb586ec84ad952483186b69cb29b ht.209417 ({"error":"timeout"})
+//MERKLE DIDNT VERIFY.ZEC 2520e53f9e0366f711bd924fcfc6fa5c3de7f095e8d7459a68daba2d2124b262 ht.209418 ({"error":"timeout"})
 // pricearray? RT metrics
 // select oldest utxo first, handles <-> pubkeys, reputations, bonds etc.
 //
@@ -105,6 +107,7 @@ char *blocktrail_listtransactions(char *symbol,char *coinaddr,int32_t num,int32_
 #include "LP_utxos.c"
 #include "LP_forwarding.c"
 #include "LP_signatures.c"
+#include "LP_RTmetrics.c"
 #include "LP_ordermatch.c"
 #include "LP_portfolio.c"
 #include "LP_messages.c"
@@ -742,12 +745,16 @@ void LP_price_broadcastloop(void *ctx)
         for (baseind=0; baseind<LP_MAXPRICEINFOS; baseind++)
         {
             basepp = LP_priceinfo(baseind);
+            if ( basepp->symbol[0] == 0 )
+                continue;
             for (relind=0; relind<LP_MAXPRICEINFOS; relind++)
             {
                 relpp = LP_priceinfo(relind);
+                if ( relpp->symbol[0] == 0 )
+                    continue;
                 if ( basepp != 0 && relpp != 0 && (price= relpp->myprices[basepp->ind]) > SMALLVAL)
                 {
-                    printf("automated price broadcast %s/%s %.8f\n",relpp->symbol,basepp->symbol,price);
+                    //printf("automated price broadcast %s/%s %.8f\n",relpp->symbol,basepp->symbol,price);
                     LP_pricepings(ctx,LP_myipaddr,LP_mypubsock,relpp->symbol,basepp->symbol,price);
                 }
             }
@@ -1095,14 +1102,14 @@ void LP_fromjs_iter()
         ctx = bitcoin_ctx();
     if ( 0 && (LP_counter % 100) == 0 )
         printf("LP_fromjs_iter got called LP_counter.%d userpass.(%s) ctx.%p\n",LP_counter,G.USERPASS,ctx);
-    //if ( Nanomsg_threadarg != 0 )
+    if ( Nanomsg_threadarg != 0 )
-    //    nn_thread_main_routine(Nanomsg_threadarg);
+        nn_thread_main_routine(Nanomsg_threadarg);
     //LP_pubkeys_query();
-    LP_utxosQ_process();
+    //LP_utxosQ_process();
     LP_nanomsg_recvs(ctx);
-    LP_mainloop_iter(ctx,LP_myipaddr,0,LP_mypubsock,LP_publicaddr,LP_RPCPORT);
+    //LP_mainloop_iter(ctx,LP_myipaddr,0,LP_mypubsock,LP_publicaddr,LP_RPCPORT);
-    queue_loop(0);
+    //queue_loop(0);
-    if ( (LP_counter % 10) == 0 ) // 10 seconds
+    if ( 0 && (LP_counter % 10) == 0 ) // 10 seconds
     {
         LP_coinsloop(0);
         if ( (LP_counter % 100) == 0 ) // 100 seconds

iguana/exchanges/LP_network.c

@@ -28,6 +28,73 @@ struct psock
 
 uint16_t Numpsocks,Psockport = MIN_PSOCK_PORT;
 
+#ifdef FROM_JS
+/*
+int32_t nn_socket(int domain, int protocol)
+{
+    return(0);
+}
+
+int32_t nn_close(int s)
+{
+    return(0);
+}
+
+int32_t nn_setsockopt(int s, int level, int option, const void *optval,size_t optvallen)
+{
+    return(0);
+}
+
+int32_t nn_getsockopt(int s, int level, int option, void *optval,size_t *optvallen)
+{
+    return(0);
+}
+
+int32_t nn_bind(int s, const char *addr)
+{
+    return(-1);
+}
+
+int32_t nn_connect(int s, const char *addr)
+{
+    if ( strncmp("ws://",addr,strlen("ws://")) != 0 )
+        return(-1);
+    return(0);
+}
+
+int32_t nn_shutdown(int s, int how)
+{
+    return(0);
+}
+
+int32_t nn_send(int s, const void *buf, size_t len, int flags)
+{
+    return(0);
+}
+
+int32_t nn_recv(int s, void *buf, size_t len, int flags)
+{
+    return(0);
+}
+
+int32_t nn_errno(void)
+{
+    return(-11);
+}
+
+const char *nn_strerror(int errnum)
+{
+    return("nanomsg error");
+}
+
+int32_t nn_poll(struct nn_pollfd *fds, int nfds, int timeout)
+{
+    return(0);
+}*/
+
+
+#endif
+
 char *nanomsg_transportname(int32_t bindflag,char *str,char *ipaddr,uint16_t port)
 {
     sprintf(str,"ws://%s:%u",bindflag == 0 ? ipaddr : "*",port);

iguana/exchanges/LP_ordermatch.c

@@ -766,7 +766,7 @@ struct LP_utxoinfo *LP_ordermatch_iter(struct LP_address_utxo **utxos,int32_t ma
 
 struct LP_utxoinfo *LP_buyutxo(double *ordermatchpricep,int64_t *bestsatoshisp,int64_t *bestdestsatoshisp,struct LP_utxoinfo *autxo,char *base,double maxprice,int32_t duration,uint64_t txfee,uint64_t desttxfee,char *gui,bits256 *avoids,int32_t numavoids,bits256 destpubkey)
 {
-    bits256 pubkey; char *obookstr,coinaddr[64]; cJSON *orderbook,*asks,*item; int32_t maxiters,i,j,numasks,max; struct LP_address_utxo **utxos; double price; struct LP_pubkeyinfo *pubp; uint64_t asatoshis; struct iguana_info *basecoin; struct LP_utxoinfo *bestutxo = 0;
+    bits256 pubkey; char *obookstr,coinaddr[64]; cJSON *orderbook,*asks,*rawasks,*item; int32_t maxiters,i,j,numasks,max; struct LP_address_utxo **utxos; double price; struct LP_pubkeyinfo *pubp; uint64_t asatoshis; struct iguana_info *basecoin; struct LP_utxoinfo *bestutxo = 0;
     maxiters = 100;
     *ordermatchpricep = 0.;
     *bestsatoshisp = *bestdestsatoshisp = 0;
@@ -785,8 +785,10 @@ struct LP_utxoinfo *LP_buyutxo(double *ordermatchpricep,int64_t *bestsatoshisp,i
     {
         if ( (orderbook= cJSON_Parse(obookstr)) != 0 )
         {
-            if ( (asks= jarray(&numasks,orderbook,"asks")) != 0 )
+            if ( (rawasks= jarray(&numasks,orderbook,"asks")) != 0 )
             {
+                if ( (asks= LP_RTmetrics_sort(rawasks,numasks,maxprice,dstr(autxo->S.satoshis))) == 0 )
+                    asks = rawasks;
                 for (i=0; i<numasks; i++)
                 {
                     item = jitem(asks,i);
@@ -832,6 +834,8 @@ struct LP_utxoinfo *LP_buyutxo(double *ordermatchpricep,int64_t *bestsatoshisp,i
                         break;
                     }
                 }
+                if ( asks != 0 && asks != rawasks )
+                    free_json(asks);
             }
             free_json(orderbook);
         }

iguana/exchanges/LP_peers.c

@@ -90,7 +90,7 @@ struct LP_peerinfo *LP_addpeer(struct LP_peerinfo *mypeer,int32_t mypubsock,char
             if ( pushport != 0 && subport != 0 && (pushsock= nn_socket(AF_SP,NN_PUSH)) >= 0 )
             {
                 nanomsg_transportname(0,pushaddr,peer->ipaddr,pushport);
-                nanomsg_transportname(0,pushaddr2,peer->ipaddr,pushport);
+                nanomsg_transportname2(0,pushaddr2,peer->ipaddr,pushport);
                 valid = 0;
                 if ( nn_connect(pushsock,pushaddr) >= 0 )
                     valid++;

iguana/exchanges/LP_prices.c

@@ -197,7 +197,7 @@ struct LP_pubkeyinfo *LP_pubkeyfind(bits256 pubkey)
 
 struct LP_pubkeyinfo *LP_pubkeyadd(bits256 pubkey)
 {
-    struct LP_pubkeyinfo *pubp=0;
+    char str[65]; struct LP_pubkeyinfo *pubp=0;
     portable_mutex_lock(&LP_pubkeymutex);
     HASH_FIND(hh,LP_pubkeyinfos,&pubkey,sizeof(pubkey),pubp);
     if ( pubp == 0 )
@@ -212,7 +212,7 @@ struct LP_pubkeyinfo *LP_pubkeyadd(bits256 pubkey)
         HASH_ADD_KEYPTR(hh,LP_pubkeyinfos,&pubp->pubkey,sizeof(pubp->pubkey),pubp);
         HASH_FIND(hh,LP_pubkeyinfos,&pubkey,sizeof(pubkey),pubp);
         if ( pubp == 0 )
-            printf("pubkeyadd find error after add\n");
+            printf("pubkeyadd find %s error after add\n",bits256_str(str,pubp->pubkey));
     }
     portable_mutex_unlock(&LP_pubkeymutex);
     return(pubp);

iguana/exchanges/LP_rpc.c

@@ -108,7 +108,7 @@ char *LP_apicall(struct iguana_info *coin,char *method,char *params)
         return(0);
     if ( coin->electrum != 0 )
     {
-        if ( (retjson= electrum_submit(coin->symbol,coin->electrum,&retjson,method,params,LP_HTTP_TIMEOUT)) != 0 )
+        if ( (retjson= electrum_submit(coin->symbol,coin->electrum,&retjson,method,params,ELECTRUM_TIMEOUT)) != 0 )
         {
             retstr = jprint(retjson,0);
             //printf("got.%p (%s)\n",retjson,retstr);
@@ -142,7 +142,7 @@ cJSON *bitcoin_json(struct iguana_info *coin,char *method,char *params)
             }
             else
             {
-                if ( (retjson= electrum_submit(coin->symbol,coin->electrum,&retjson,method,params,LP_HTTP_TIMEOUT)) != 0 )
+                if ( (retjson= electrum_submit(coin->symbol,coin->electrum,&retjson,method,params,ELECTRUM_TIMEOUT)) != 0 )
                 {
                     if ( jobj(retjson,"error") != 0 )
                     {

iguana/exchanges/LP_signatures.c

@@ -204,7 +204,7 @@ char *LP_quotereceived(cJSON *argjson)
 
 int32_t LP_bitcoinsig_add(cJSON *item,bits256 priv,uint8_t *pubsecp,bits256 sighash)
 {
-    static void *ctx; int32_t i,j,siglen; uint8_t pub33[33],sig[65]; char sigstr[128];
+    static void *ctx; int32_t i,j,siglen; uint8_t pub33[33],sig[65]; char sigstr[256];
     if ( ctx == 0 )
         ctx = bitcoin_ctx();
     for (j=0; j<100; j++)

iguana/exchanges/LP_socket.c

@@ -243,7 +243,7 @@ struct electrum_info
     queue_t sendQ,pendingQ;
     portable_mutex_t mutex,txmutex;
     struct electrum_info *prev;
-    int32_t bufsize,sock,*heightp;
+    int32_t bufsize,sock,*heightp,numerrors;
     struct iguana_info *coin;
     uint32_t stratumid,lasttime,pending,*heighttimep;
     char ipaddr[64],symbol[16];
@@ -382,8 +382,23 @@ cJSON *electrum_submit(char *symbol,struct electrum_info *ep,cJSON **retjsonp,ch
             queue_enqueue("sendQ",&ep->sendQ,&sitem->DL);
             expiration = (uint32_t)time(NULL) + timeout + 1;
             while ( *retjsonp == 0 && time(NULL) <= expiration )
-                usleep(10000);
+                usleep(5000);
             portable_mutex_unlock(&ep->mutex);
+            if ( *retjsonp == 0 || jobj(*retjsonp,"error") != 0 )
+            {
+                if ( ++ep->numerrors >= LP_ELECTRUM_MAXERRORS )
+                {
+                    closesocket(ep->sock), ep->sock = -1;
+                    if ( (ep->sock= LP_socket(0,ep->ipaddr,ep->port)) < 0 )
+                        printf("error RE-connecting to %s:%u\n",ep->ipaddr,ep->port);
+                    else
+                    {
+                        printf("ep.%p %s numerrors.%d too big -> new %s:%u sock.%d\n",ep,ep->symbol,ep->numerrors,ep->ipaddr,ep->port,ep->sock);
+                        ep->numerrors = 0;
+                    }
+                }
+            } else if ( ep->numerrors > 0 )
+                ep->numerrors++;
             if ( ep->prev == 0 )
             {
                 if ( *retjsonp == 0 )
@@ -395,6 +410,8 @@ cJSON *electrum_submit(char *symbol,struct electrum_info *ep,cJSON **retjsonp,ch
             }
         } else printf("couldnt find electrum server for (%s %s) or no retjsonp.%p\n",method,params,retjsonp);
         ep = ep->prev;
+        if ( ep != 0 )
+            printf("using prev ep.%s\n",ep->symbol);
     }
     return(0);
 }
@@ -898,24 +915,6 @@ void LP_dedicatedloop(void *arg)
             if ( sitem->expiration != 0 )
                 sitem->expiration += (uint32_t)time(NULL);
             else sitem->expiration = (uint32_t)time(NULL) + ELECTRUM_TIMEOUT;
-            /*portable_mutex_lock(&ep->pendingQ.mutex);
-            if ( ep->pendingQ.list != 0 )
-            {
-                printf("list %p\n",ep->pendingQ.list);
-                DL_FOREACH_SAFE(ep->pendingQ.list,item,tmp)
-                {
-                    printf("item.%p\n",item);
-                    if ( item->type == 0xffffffff )
-                    {
-                        printf("%p purge %s",item,((struct stritem *)item)->str);
-                        DL_DELETE(ep->pendingQ.list,item);
-                        free(item);
-                    }
-                }
-            }
-            DL_APPEND(ep->pendingQ.list,&sitem->DL);
-            portable_mutex_unlock(&ep->pendingQ.mutex);*/
-            //printf("%p SENT.(%s) to %s:%u\n",sitem,sitem->str,ep->ipaddr,ep->port);
             queue_enqueue("pendingQ",&ep->pendingQ,&sitem->DL);
             flag++;
         }

iguana/exchanges/LP_statemachine.c

@@ -2349,6 +2349,24 @@ struct LP_utxoinfo *LP_bestutxo(double *ordermatchpricep,int64_t *bestsatoshisp,
     LP_mypriceset(&changed,autxo->coin,base,1. / *ordermatchpricep);
     return(bestutxo);
 }
+/*portable_mutex_lock(&ep->pendingQ.mutex);
+ if ( ep->pendingQ.list != 0 )
+ {
+ printf("list %p\n",ep->pendingQ.list);
+ DL_FOREACH_SAFE(ep->pendingQ.list,item,tmp)
+ {
+ printf("item.%p\n",item);
+ if ( item->type == 0xffffffff )
+ {
+ printf("%p purge %s",item,((struct stritem *)item)->str);
+ DL_DELETE(ep->pendingQ.list,item);
+ free(item);
+ }
+ }
+ }
+ DL_APPEND(ep->pendingQ.list,&sitem->DL);
+ portable_mutex_unlock(&ep->pendingQ.mutex);*/
+//printf("%p SENT.(%s) to %s:%u\n",sitem,sitem->str,ep->ipaddr,ep->port);
 
 char *LP_ordermatch(char *base,int64_t txfee,double maxprice,double maxvolume,char *rel,bits256 txid,int32_t vout,bits256 feetxid,int32_t feevout,int64_t desttxfee,int32_t duration)
 {

iguana/exchanges/Makefile

@@ -1,10 +1,10 @@
 #cd ..
-#emcc -s ASSERTIONS=1 -s ASYNCIFY=1 -s TOTAL_MEMORY=134217728 --preload-file coins.json -DFROM_JS -O2 -I../includes -I../crypto777 -s PTHREAD_POOL_SIZE=8 -s USE_PTHREADS=1 -o /var/www/html/index.html exchanges/mm.c ../crypto777/*.c ../crypto777/jpeg/*.c ../crypto777/jpeg/unix/*.c mini-gmp.c secp256k1/src/secp256k1.c -lm ../OSlibs/js/libnanomsg.a -lc -lpthread
 
 #emcc -s ASSERTIONS=1 -s ASYNCIFY=1 -s TOTAL_MEMORY=134217728 --preload-file coins.json -D__PNACL -DFROM_JS -O2 -I../includes -I../crypto777 -s PTHREAD_POOL_SIZE=8 -s USE_PTHREADS=1 -o /var/www/html/index.html exchanges/mm.c $(CRYPTO777_SRCS) mini-gmp.c secp256k1/src/secp256k1.c -lm -lc
 
-include ../crypto777/crypto777.sources
+#include ../crypto777/crypto777.sources
+#emcc -s ASSERTIONS=1 -s ASYNCIFY=1 -s TOTAL_MEMORY=134217728 --preload-file coins.json -D__PNACL -DFROM_JS -O2 -I../includes -I../crypto777 -o /var/www/html/index.html exchanges/mm.c $(CRYPTO777_SRCS) mini-gmp.c secp256k1/src/secp256k1.c -lm -lc
 
 all:
-	emcc -s ASSERTIONS=1 -s ASYNCIFY=1 -s TOTAL_MEMORY=134217728 --preload-file coins.json -D__PNACL -DFROM_JS -O2 -I../includes -I../crypto777 -o /var/www/html/index.html exchanges/mm.c $(CRYPTO777_SRCS) mini-gmp.c secp256k1/src/secp256k1.c -lm -lc
+	emcc -s PTHREAD_POOL_SIZE=8 -s USE_PTHREADS=1 -s ABORTING_MALLOC=0 -s ASSERTIONS=1 -s ASYNCIFY=1 -s TOTAL_MEMORY=536870912 --preload-file coins.json -DFROM_JS -O2 -I../includes -I../crypto777 -s PTHREAD_POOL_SIZE=8 -s USE_PTHREADS=1 -o /var/www/html/index.html exchanges/mm.c ../crypto777/*.c ../crypto777/jpeg/*.c ../crypto777/jpeg/unix/*.c mini-gmp.c secp256k1/src/secp256k1.c -lm -lpthread
 

iguana/exchanges/nodeinstall

@@ -0,0 +1,16 @@
+cd /var/www/html/
+node install http-server -g
+
+#start http-server to start current directory serving via http://127.0.0.1:8080
+http-server .
+
+node install node-nanomsg
+
+# test.js
+#var nano = require('nanomsg');
+#var sub = nano.socket('sub');
+#var addr = 'ws://5.9.253.197:7785'
+#sub.connect(addr);
+#sub.on('data', function (buf) {
+#console.log(String(buf));
+#});