RTmetrics

iguana/exchanges/LP_RTmetrics.c

@@ -18,9 +18,139 @@
 //  marketmaker
 //
 
-cJSON *LP_RTmetrics_sort(cJSON *rawasks,int32_t numasks,double maxprice,double relvolume)
+struct LP_metricinfo
 {
-    cJSON *array,*item,*statsjson,*swaps=0; int32_t i,numswaps=0; bits256 zero; uint32_t futuretime; double price; char *retstr;
+    bits256 pubkey;
+    double metric,price,balance,minvol,maxvol;
+    int32_t ind,numutxos,age,pendingswaps;
+};
+
+struct LP_RTmetrics_pendings
+{
+    char refbase[16],refrel[16];
+    int32_t numswaps,numavoidtxids,numwhitelist,numblacklist,numpendings,pending_swaps[1024];
+    bits256 avoidtxids[8192],whitelist[1024],blacklist[1024],pending_pubkeys[1024];
+} LP_RTmetrics;
+
+int32_t LP_bits256_find(bits256 *list,int32_t num,bits256 val)
+{
+    int32_t i;
+    if ( bits256_nonz(val) != 0 )
+    {
+        for (i=0; i<num; i++)
+            if ( bits256_cmp(list[i],val) == 0 )
+                return(i);
+    }
+    return(-1);
+}
+
+int32_t LP_bits256_add(char *debugstr,bits256 *list,int32_t *nump,int32_t maxnum,bits256 val)
+{
+    if ( bits256_nonz(val) != 0 && *nump < maxnum )
+    {
+        if ( LP_bits256_find(list,*nump,val) < 0 )
+            list[(*nump)++] = val;
+        return(*nump);
+    } else printf("%s[%d] overflow\n",debugstr,*nump);
+    return(-1);
+}
+
+int32_t LP_RTmetrics_avoidadd(bits256 txid)
+{
+    return(LP_bits256_add("LP_RTmetrics_avoidadd avoidtxids",LP_RTmetrics.avoidtxids,&LP_RTmetrics.numavoidtxids,(int32_t)(sizeof(LP_RTmetrics.avoidtxids)/sizeof(*LP_RTmetrics.avoidtxids)),txid));
+}
+
+int32_t LP_RTmetrics_whitelistadd(bits256 pubkey)
+{
+    return(LP_bits256_add("LP_RTmetrics_whitelistadd whitelist",LP_RTmetrics.whitelist,&LP_RTmetrics.numwhitelist,(int32_t)(sizeof(LP_RTmetrics.whitelist)/sizeof(*LP_RTmetrics.whitelist)),pubkey));
+}
+
+int32_t LP_RTmetrics_blacklistadd(bits256 pubkey)
+{
+    return(LP_bits256_add("LP_RTmetrics_blacklistadd blacklist",LP_RTmetrics.blacklist,&LP_RTmetrics.numblacklist,(int32_t)(sizeof(LP_RTmetrics.blacklist)/sizeof(*LP_RTmetrics.blacklist)),pubkey));
+}
+
+int32_t LP_RTmetrics_pendingswap(bits256 pubkey)
+{
+    int32_t ind;
+    if ( (ind= LP_bits256_add("LP_RTmetrics_pendingswap",LP_RTmetrics.pending_pubkeys,&LP_RTmetrics.numpendings,(int32_t)(sizeof(LP_RTmetrics.pending_pubkeys)/sizeof(*LP_RTmetrics.pending_pubkeys)),pubkey)) >= 0 )
+        LP_RTmetrics.pending_swaps[ind]++;
+    return(ind);
+}
+
+int32_t LP_RTmetrics_pendingswaps(bits256 pubkey)
+{
+    int32_t ind;
+    if ( (ind= LP_bits256_find(LP_RTmetrics.pending_pubkeys,LP_RTmetrics.numpendings,pubkey)) >= 0 )
+        return(LP_RTmetrics.pending_swaps[ind]);
+    else return(0);
+}
+
+int32_t LP_RTmetrics_avoidtxid(bits256 txid)
+{
+    return(LP_bits256_find(LP_RTmetrics.avoidtxids,LP_RTmetrics.numavoidtxids,txid));
+}
+
+int32_t LP_RTmetrics_whitelisted(bits256 pubkey)
+{
+    return(LP_bits256_find(LP_RTmetrics.whitelist,LP_RTmetrics.numwhitelist,pubkey));
+}
+
+int32_t LP_RTmetrics_blacklisted(bits256 pubkey)
+{
+    return(LP_bits256_find(LP_RTmetrics.blacklist,LP_RTmetrics.numblacklist,pubkey));
+}
+
+void LP_RTmetrics_swapsinfo(char *refbase,char *refrel,cJSON *swaps,int32_t numswaps)
+{
+    int32_t i; char *base,*rel,*retstr; cJSON *item,*swapjson; bits256 srcpub,destpub; uint64_t aliceid,basesatoshis,relsatoshis; uint32_t requestid,quoteid; double price;
+    for (i=0; i<numswaps; i++)
+    {
+        item = jitem(swaps,i);
+        if ( (base= jstr(item,"base")) == 0 )
+            base = "";
+        if ( (rel= jstr(item,"rel")) == 0 )
+            rel = "";
+        if ( strcmp(base,refbase) != 0 && strcmp(base,refrel) != 0 && strcmp(rel,refbase) != 0 && strcmp(rel,refrel) != 0 )
+            continue;
+        aliceid = j64bits(item,"aliceid");
+        basesatoshis = SATOSHIDEN * jdouble(item,"basevol");
+        srcpub = jbits256(item,"src");
+        relsatoshis = SATOSHIDEN * jdouble(item,"relvol");
+        destpub = jbits256(item,"dest");
+        price = jdouble(item,"price");
+        requestid = juint(item,"requestid");
+        quoteid = juint(item,"quoteid");
+        LP_RTmetrics_pendingswap(srcpub);
+        LP_RTmetrics_pendingswap(destpub);
+        if ( (retstr= basilisk_swapentry(requestid,quoteid)) != 0 )
+        {
+            if ( (swapjson= cJSON_Parse(retstr)) != 0 )
+            {
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"bobdeposit"));
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"alicepayment"));
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"bobpayment"));
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"paymentspent"));
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"Apaymentspent"));
+                LP_RTmetrics_avoidadd(jbits256(swapjson,"depositspent"));
+                free_json(swapjson);
+            }
+            free(retstr);
+        }
+    }
+}
+
+void LP_RTmetrics_update(char *base,char *rel)
+{
+    struct LP_pubkeyinfo *pubp,*tmp; uint32_t futuretime; int32_t i,numswaps; bits256 zero; char *retstr; cJSON *statsjson,*swaps;
+    memset(&LP_RTmetrics,0,sizeof(LP_RTmetrics));
+    HASH_ITER(hh,LP_pubkeyinfos,pubp,tmp)
+    {
+        if ( pubp->istrusted > 0 )
+            LP_RTmetrics_whitelistadd(pubp->pubkey);
+        else if ( pubp->istrusted < 0 )
+            LP_RTmetrics_blacklistadd(pubp->pubkey);
+    }
     futuretime = (uint32_t)time(NULL) + 3600*100;
     memset(zero.bytes,0,sizeof(zero));
     if ( (retstr= LP_statslog_disp(100,futuretime,futuretime,"",zero)) != 0 )
@@ -29,37 +159,110 @@ cJSON *LP_RTmetrics_sort(cJSON *rawasks,int32_t numasks,double maxprice,double r
         {
             if ( (swaps= jarray(&numswaps,statsjson,"swaps")) != 0 )
             {
+                printf("LP_RTmetrics_update for (%s)\n",jprint(swaps,0));
                 if ( numswaps > 0 )
-                    swaps = jduplicate(swaps);
-                else swaps = 0;
+                    LP_RTmetrics_swapsinfo(base,rel,swaps,numswaps);
             }
             free_json(statsjson);
         }
         free(retstr);
     }
-    //if ( numswaps == 0 || swaps == 0 )
-        return(0);
-    printf("calc RTmetrics for (%s)\n",jprint(swaps,0));
-    /*jadd64bits(item,"aliceid",sp->aliceid);
-    jaddbits256(item,"src",sp->Q.srchash);
-    jaddstr(item,"base",sp->Q.srccoin);
-    jaddnum(item,"basevol",dstr(sp->Q.satoshis));
-    jaddbits256(item,"dest",sp->Q.desthash);
-    jaddstr(item,"rel",sp->Q.destcoin);
-    jaddnum(item,"relvol",dstr(sp->Q.destsatoshis));
-    jaddnum(item,"price",sp->qprice);
-    jaddnum(item,"requestid",sp->Q.R.requestid);
-    jaddnum(item,"quoteid",sp->Q.R.quoteid);
-    */
-    array = cJSON_CreateArray();
-    for (i=0; i<numasks; i++)
+    for (i=0; i<LP_RTmetrics.numpendings; i++)
+        if ( LP_RTmetrics.pending_swaps[i] > LP_MAXPENDING_SWAPS )
+        {
+            char str[65]; printf("%s has %d pending swaps! which is more than %d\n",bits256_str(str,LP_RTmetrics.pending_pubkeys[i]),LP_RTmetrics.pending_swaps[i],LP_MAXPENDING_SWAPS);
+            LP_RTmetrics_blacklistadd(LP_RTmetrics.pending_pubkeys[i]);
+        }
+    printf("%d pubkeys have pending swaps, whitelist.%d blacklist.%d avoidtxids.%d\n",LP_RTmetrics.numpendings,LP_RTmetrics.numwhitelist,LP_RTmetrics.numblacklist,LP_RTmetrics.numavoidtxids);
+}
+
+double _LP_RTmetric_calc(struct LP_metricinfo *mp,double bestprice,double maxprice,double relvolume)
+{
+    int32_t n; double metric,origmetric = (bestprice / mp->price);
+    metric = origmetric;
+    if ( mp->numutxos == 0 || relvolume == 0. || mp->maxvol == 0. || mp->balance == 0. )
+        return(metric * 100.);
+    if ( relvolume < mp->minvol )
+    {
+        metric *= (mp->minvol / relvolume);
+    }
+    else if ( relvolume > mp->maxvol )
+    {
+        metric *= (relvolume / mp->maxvol);
+    }
+    if ( relvolume < mp->balance/LP_MINVOL )
+    {
+        metric *= (mp->balance / relvolume);
+    }
+    else if ( relvolume > mp->balance/mp->numutxos )
+    {
+        metric *= (relvolume / (mp->balance/mp->numutxos));
+    }
+    if ( mp->age > LP_ORDERBOOK_DURATION*0.8 )
+        metric *= 2;
+    else if ( mp->age > 60 )
+        metric *= 1.03;
+    if ( (n= mp->pendingswaps) > 0 )
+        while ( n-- > 0 )
+            metric *= 1.1;
+    if ( metric != origmetric )
+        printf("price %.8f orig %.8f -> %.8f relvol %.8f min %.8f max %.8f bal %.8f age.%d pend.%d\n",mp->price,origmetric,metric,relvolume,mp->minvol,mp->maxvol,mp->balance,mp->age,mp->pendingswaps);
+    return(metric);
+}
+
+void LP_RTmetric_calc(struct LP_metricinfo *sortbuf,int32_t ind,cJSON *item,double bestprice,double maxprice,double relvolume,double prevdepth)
+{
+    sortbuf[ind].pubkey = jbits256(item,"pubkey");
+    sortbuf[ind].price = jdouble(item,"price");
+    sortbuf[ind].maxvol = jdouble(item,"maxvolume");
+    sortbuf[ind].minvol = jdouble(item,"minvolume");
+    sortbuf[ind].balance = jdouble(item,"depth") - prevdepth;
+    sortbuf[ind].numutxos = juint(item,"numutxos");
+    sortbuf[ind].age = juint(item,"age");
+    sortbuf[ind].ind = ind;
+    sortbuf[ind].pendingswaps = LP_RTmetrics_pendingswaps(sortbuf[ind].pubkey);
+    sortbuf[ind].metric = _LP_RTmetric_calc(&sortbuf[ind],bestprice,maxprice,relvolume);
+}
+
+cJSON *LP_RTmetrics_sort(char *base,char *rel,cJSON *rawasks,int32_t numasks,double maxprice,double relvolume)
+{
+    cJSON *array,*item; int32_t i,num,groupi; double price,prevdepth,bestprice; struct LP_metricinfo *sortbuf;
+    groupi = -1;
+    bestprice = 0.;
+    for (num=i=0; i<numasks; i++)
     {
         item = jitem(rawasks,i);
         price = jdouble(item,"price");
         if ( price > maxprice )
             break;
-        jaddi(array,jduplicate(item));
+        if ( i == 0 )
+            bestprice = price;
+        else if ( price < bestprice*1.01 )
+            groupi = i;
+        num++;
     }
-    free_json(swaps);
+    if ( groupi > 0 )
+    {
+        sortbuf = calloc(groupi+1,sizeof(*sortbuf));
+        prevdepth = 0.;
+        for (i=0; i<=groupi; i++)
+        {
+            item = jitem(rawasks,i);
+            LP_RTmetric_calc(sortbuf,i,item,bestprice,maxprice,relvolume,prevdepth);
+            prevdepth = jdouble(item,"depth");
+        }
+        revsortds(&sortbuf[0].metric,groupi+1,sizeof(*sortbuf));
+        array = cJSON_CreateArray();
+        for (i=0; i<=groupi; i++)
+        {
+            printf("(%d -> %d) ",i,sortbuf[i].ind);
+            item = jitem(rawasks,sortbuf[i].ind);
+            jaddi(array,jduplicate(item));
+        }
+        free(sortbuf);
+        for (; i<num; i++)
+            jaddi(array,jduplicate(jitem(rawasks,i)));
+        printf("new ask order for %d of %d, capped at num.%d\n",groupi,numasks,num);
+    } else array = rawasks;
     return(array);
 }

iguana/exchanges/LP_commands.c

@@ -128,7 +128,7 @@ getpeers()\n\
 passphrase(passphrase, gui)\n\
 listunspent(coin, address)\n\
 setconfirms(coin, numconfirms, maxconfirms=6)\n\
-trust(pubkey, trust)\n\
+trust(pubkey, trust) # positive to trust, 0 for normal, negative to blacklist\n\
 balance(coin, address)\n\
 orderbook(base, rel, duration=3600)\n\
 getprices(base, rel)\n\

iguana/exchanges/LP_include.h

@@ -39,6 +39,7 @@ void emscripten_usleep(int32_t x);
 #define ELECTRUM_TIMEOUT 10
 #define LP_ELECTRUM_MAXERRORS 3
 #define LP_MEMPOOL_TIMEINCR 10
+#define LP_MAXPENDING_SWAPS 13
 
 #define LP_COMMAND_SENDSOCK NN_PUSH
 #define LP_COMMAND_RECVSOCK NN_PULL
@@ -52,7 +53,7 @@ void emscripten_usleep(int32_t x);
 #define MAX_PSOCK_PORT 60000
 #define MIN_PSOCK_PORT 10000
 #define LP_GETINFO_INCR 30
-#define LP_ORDERBOOK_DURATION 120
+#define LP_ORDERBOOK_DURATION 180
 
 #define LP_MAXPEER_ERRORS 3
 #define LP_MINPEER_GOOD 20
@@ -313,7 +314,8 @@ struct LP_pubkeyinfo
     UT_hash_handle hh;
     bits256 pubkey;
     double matrix[LP_MAXPRICEINFOS][LP_MAXPRICEINFOS];
-    uint32_t timestamp,istrusted,numerrors;
+    uint32_t timestamp,numerrors;
+    int32_t istrusted;
     uint8_t rmd160[20],sig[65],pubsecp[33],siglen;
 };
 
@@ -361,11 +363,13 @@ int32_t iguana_signrawtransaction(void *ctx,char *symbol,uint8_t wiftaddr,uint8_
 struct LP_address_utxo *LP_address_utxofind(struct iguana_info *coin,char *coinaddr,bits256 txid,int32_t vout);
 int32_t LP_destaddr(char *destaddr,cJSON *item);
 int32_t LP_waitmempool(char *symbol,char *coinaddr,bits256 txid,int32_t vout,int32_t duration);
+char *LP_statslog_disp(int32_t n,uint32_t starttime,uint32_t endtime,char *refgui,bits256 refpubkey);
 struct LP_transaction *LP_transactionfind(struct iguana_info *coin,bits256 txid);
 cJSON *LP_transactioninit(struct iguana_info *coin,bits256 txid,int32_t iter,cJSON *txobj);
 int32_t LP_mempoolscan(char *symbol,bits256 searchtxid);
 int32_t LP_txheight(struct iguana_info *coin,bits256 txid);
 int32_t LP_numpeers();
+char *basilisk_swapentry(uint32_t requestid,uint32_t quoteid);
 uint64_t LP_KMDvalue(struct iguana_info *coin,uint64_t balance);
 int32_t LP_address_utxoadd(char *debug,struct iguana_info *coin,char *coinaddr,bits256 txid,int32_t vout,uint64_t value,int32_t height,int32_t spendheight);
 void LP_smartutxos_push(struct iguana_info *coin);

iguana/exchanges/LP_nativeDEX.c

@@ -18,8 +18,6 @@
 //  LP_nativeDEX.c
 //  marketmaker
 //
-//MERKLE DIDNT VERIFY.ZEC 96d8484efb1f96e2a692f572d2b3bbc49a59bb586ec84ad952483186b69cb29b ht.209417 ({"error":"timeout"})
-//MERKLE DIDNT VERIFY.ZEC 2520e53f9e0366f711bd924fcfc6fa5c3de7f095e8d7459a68daba2d2124b262 ht.209418 ({"error":"timeout"})
 // pricearray? RT metrics
 // select oldest utxo first, handles <-> pubkeys, reputations, bonds etc.
 //
@@ -96,6 +94,7 @@ char *blocktrail_listtransactions(char *symbol,char *coinaddr,int32_t num,int32_
 #include "LP_bitcoin.c"
 #include "LP_coins.c"
 #include "LP_rpc.c"
+#include "LP_RTmetrics.c"
 #include "LP_utxo.c"
 #include "LP_prices.c"
 #include "LP_scan.c"
@@ -107,7 +106,6 @@ char *blocktrail_listtransactions(char *symbol,char *coinaddr,int32_t num,int32_
 #include "LP_utxos.c"
 #include "LP_forwarding.c"
 #include "LP_signatures.c"
-#include "LP_RTmetrics.c"
 #include "LP_ordermatch.c"
 #include "LP_portfolio.c"
 #include "LP_messages.c"
@@ -732,7 +730,7 @@ void LP_pubkeysloop(void *ctx)
         LP_counter += 100;
         //printf("LP_pubkeysloop %d\n",LP_counter);
         LP_notify_pubkeys(ctx,LP_mypubsock);
-        sleep(60);
+        sleep(LP_ORDERBOOK_DURATION * .777);
     }
 }
 
@@ -771,7 +769,7 @@ void LP_privkeysloop(void *ctx)
         LP_counter += 1000;
         //printf("LP_privkeysloop %u\n",LP_counter);
         LP_privkey_updates(ctx,LP_mypubsock,0);
-        sleep(60);
+        sleep(LP_ORDERBOOK_DURATION * .777);
     }
 }
 

iguana/exchanges/LP_ordermatch.c

@@ -656,6 +656,12 @@ int32_t LP_tradecommand(void *ctx,char *myipaddr,int32_t pubsock,cJSON *argjson,
                 printf("request.(%s)\n",jprint(argjson,0));
                 if ( 1 )//LP_allocated(butxo->payment.txid,butxo->payment.vout) != 0 || LP_allocated(butxo->deposit.txid,butxo->deposit.vout) != 0 || (qprice= LP_quote_validate(autxo,butxo,&Q,1)) <= SMALLVAL )
                 {
+                    LP_RTmetrics_update(Q.srccoin,Q.destcoin);
+                    if ( LP_RTmetrics_blacklisted(Q.desthash) >= 0 )
+                    {
+                        printf("request from blacklisted %s, ignore\n",bits256_str(str,Q.desthash));
+                        return(retval);
+                    }
                     printf("butxo.%p replace path %p %s, %p %s, %.8f\n",butxo,LP_allocated(butxo->payment.txid,butxo->payment.vout),bits256_str(str,butxo->payment.txid),LP_allocated(butxo->deposit.txid,butxo->deposit.vout),bits256_str(str2,butxo->deposit.txid),LP_quote_validate(autxo,butxo,&Q,1));
                     LP_listunspent_both(Q.srccoin,Q.coinaddr,0);
                     if ( (butxo= LP_address_utxopair(1,utxos,max,LP_coinfind(Q.srccoin),Q.coinaddr,Q.txfee,dstr(Q.destsatoshis),price,Q.desttxfee)) != 0 )
@@ -787,7 +793,7 @@ struct LP_utxoinfo *LP_buyutxo(double *ordermatchpricep,int64_t *bestsatoshisp,i
         {
             if ( (rawasks= jarray(&numasks,orderbook,"asks")) != 0 )
             {
-                if ( (asks= LP_RTmetrics_sort(rawasks,numasks,maxprice,dstr(autxo->S.satoshis))) == 0 )
+                if ( (asks= LP_RTmetrics_sort(base,autxo->coin,rawasks,numasks,maxprice,dstr(autxo->S.satoshis))) == 0 )
                     asks = rawasks;
                 for (i=0; i<numasks; i++)
                 {
@@ -797,6 +803,8 @@ struct LP_utxoinfo *LP_buyutxo(double *ordermatchpricep,int64_t *bestsatoshisp,i
                     pubkey = jbits256(item,"pubkey");
                     if ( bits256_nonz(destpubkey) != 0 && bits256_cmp(destpubkey,pubkey) != 0 )
                         continue;
+                    if ( LP_RTmetrics_blacklisted(pubkey) >= 0 )
+                        continue;
                     //printf("[%d/%d] %s pubcmp %d price %.8f vs maxprice %.8f\n",i,numasks,jprint(item,0),bits256_cmp(pubkey,G.LP_mypub25519),price,maxprice);
                     if ( LP_pricevalid(price) > 0 && price <= maxprice )
                     {
@@ -904,6 +912,7 @@ char *LP_autobuy(void *ctx,char *myipaddr,int32_t mypubsock,char *base,char *rel
         printf("destsatoshis %.8f vs utxo %.8f this would have triggered an quote error -13\n",dstr(destsatoshis),dstr(autxo->payment.value));
         return(clonestr("{\"error\":\"cant find alice utxo that is small enough\"}"));
     }
+    LP_RTmetrics_update(base,rel);
     while ( 1 )
     {
         if ( (bestutxo= LP_buyutxo(&ordermatchprice,&bestsatoshis,&bestdestsatoshis,autxo,base,maxprice,duration,txfee,desttxfee,gui,pubkeys,numpubs,destpubkey)) == 0 || ordermatchprice == 0. || bestdestsatoshis == 0 )

iguana/exchanges/LP_statemachine.c

@@ -2349,6 +2349,20 @@ struct LP_utxoinfo *LP_bestutxo(double *ordermatchpricep,int64_t *bestsatoshisp,
     LP_mypriceset(&changed,autxo->coin,base,1. / *ordermatchpricep);
     return(bestutxo);
 }
+
+/*static int _LP_metric_eval(const void *a,const void *b)
+ {
+ #define aptr (*(struct LP_metricinfo **)a)
+ #define bptr (*(struct LP_metricinfo **)b)
+ if ( bptr->metric > aptr->metric )
+ return(1);
+ else if ( bptr->metric < aptr->metric )
+ return(-1);
+ return(0);
+ #undef aptr
+ #undef bptr
+ }*/
+
 /*portable_mutex_lock(&ep->pendingQ.mutex);
  if ( ep->pendingQ.list != 0 )
  {

iguana/exchanges/LP_utxo.c

@@ -149,7 +149,7 @@ int32_t LP_address_utxo_ptrs(struct iguana_info *coin,int32_t iambob,struct LP_a
     DL_FOREACH_SAFE(ap->utxos,up,tmp)
     {
         //char str[65]; printf("LP_address_utxo_ptrs %s n.%d %.8f %s v%d spendheight.%d allocated.%p\n",ap->coinaddr,n,dstr(up->U.value),bits256_str(str,up->U.txid),up->U.vout,up->spendheight,LP_allocated(up->U.txid,up->U.vout));
-        if ( up->spendheight <= 0 )
+        if ( up->spendheight <= 0 && LP_RTmetrics_avoidtxid(up->U.txid) < 0 )
         {
             if ( coin->electrum == 0 )
             {