diff --git a/iguana/iguana777.c b/iguana/iguana777.c index e593617e5..a7a3c0d4a 100755 --- a/iguana/iguana777.c +++ b/iguana/iguana777.c @@ -352,26 +352,6 @@ int32_t iguana_helpertask(FILE *fp,struct OS_memspace *mem,struct OS_memspace *m return(0); } -void iguana_balancecalc(struct iguana_info *coin,struct iguana_bundle *bp,int32_t incremental) -{ - uint32_t starttime; - if ( bp->balancefinish > 1 ) - { - printf("make sure DB files have this bp.%d\n",bp->hdrsi); - iguana_validateQ(coin,bp); - return; - } - starttime = (uint32_t)time(NULL); - if ( iguana_balancegen(coin,bp,incremental) < 0 ) - { - printf("GENERATE BALANCES ERROR ht.%d\n",bp->bundleheight); - exit(-1); - } - bp->balancefinish = (uint32_t)time(NULL); - printf("GENERATED BALANCES for ht.%d duration %d seconds\n",bp->bundleheight,bp->balancefinish - (uint32_t)starttime); - iguana_validateQ(coin,bp); -} - void iguana_helper(void *arg) { FILE *fp = 0; cJSON *argjson=0; int32_t type,helperid=rand(),flag,idle=0; @@ -435,7 +415,7 @@ void iguana_helper(void *arg) void iguana_coinloop(void *arg) { struct iguana_info *coin,**coins = arg; - struct iguana_bundle *bp; int32_t flag,i,n,bundlei; bits256 zero; char str[65]; + struct iguana_bundle *bp; int32_t flag,i,n,bundlei; bits256 zero; char str[2065]; uint32_t now; n = (int32_t)(long)coins[0]; coins++; @@ -502,6 +482,7 @@ void iguana_coinloop(void *arg) //fprintf(stderr,"metrics\n"); coin->peers.lastmetrics = iguana_updatemetrics(coin); // ranks peers } + iguana_bundlestats(coin,str); flag += iguana_processrecv(coin); if ( coin->longestchain+10000 > coin->blocks.maxbits ) iguana_recvalloc(coin,coin->longestchain + 100000); diff --git a/iguana/iguana777.h b/iguana/iguana777.h index 344c85503..9b1c792f0 100755 --- a/iguana/iguana777.h +++ b/iguana/iguana777.h @@ -797,12 +797,14 @@ int32_t iguana_Xspendmap(struct iguana_info *coin,struct iguana_ramchain *ramcha void iguana_balancesQ(struct iguana_info *coin,struct iguana_bundle *bp); int32_t iguana_balanceflush(struct iguana_info *coin,int32_t refhdrsi,int32_t purgedist); int32_t iguana_bundleissue(struct iguana_info *coin,struct iguana_bundle *bp,int32_t starti,int32_t max); -void iguana_balancecalc(struct iguana_info *coin,struct iguana_bundle *bp,int32_t incremental); +int32_t iguana_balancecalc(struct iguana_info *coin,struct iguana_bundle *bp,int32_t incremental); int32_t iguana_sendblockreqPT(struct iguana_info *coin,struct iguana_peer *addr,struct iguana_bundle *bp,int32_t bundlei,bits256 hash2,int32_t iamthreadsafe); int32_t iguana_blockreq(struct iguana_info *coin,int32_t height,int32_t priority); int64_t iguana_bundlecalcs(struct iguana_info *coin,struct iguana_bundle *bp); void iguana_ramchain_prefetch(struct iguana_info *coin,struct iguana_ramchain *ramchain); void iguana_realtime_update(struct iguana_info *coin); +int32_t iguana_mapvolatiles(struct iguana_info *coin,struct iguana_ramchain *ramchain); +void iguana_purgevolatiles(struct iguana_info *coin,struct iguana_ramchain *ramchain); extern int32_t HDRnet,netBLOCKS; diff --git a/iguana/iguana_bundles.c b/iguana/iguana_bundles.c index 988ada3f2..d651aa8f4 100755 --- a/iguana/iguana_bundles.c +++ b/iguana/iguana_bundles.c @@ -917,7 +917,7 @@ void iguana_bundlestats(struct iguana_info *coin,char *str) int32_t i,n,m,j,numv,count,pending,dispflag,numutxo,numbalances,numrecv,done,numhashes,numcached,numsaved,numemit; int64_t spaceused=0,estsize = 0; struct iguana_bundle *bp,*lastpending = 0,*firstgap = 0; struct iguana_block *block,*prev; uint32_t now; now = (uint32_t)time(NULL); - dispflag = (rand() % 1000) == 0; + dispflag = 1;//(rand() % 1000) == 0; numrecv = numhashes = numcached = numsaved = numemit = done = numutxo = numbalances = 0; count = coin->bundlescount; //sortbuf = calloc(count,sizeof(*sortbuf)*2); diff --git a/iguana/iguana_peers.c b/iguana/iguana_peers.c index 64a63ead0..fc5cd7f64 100755 --- a/iguana/iguana_peers.c +++ b/iguana/iguana_peers.c @@ -1085,7 +1085,7 @@ void iguana_dedicatedloop(struct iguana_info *coin,struct iguana_peer *addr) addr->dead = 1; } } - printf(">>>>>>>>>>>>>> finish %s dedicatedloop.%s\n",coin->symbol,addr->ipaddr); + //printf(">>>>>>>>>>>>>> finish %s dedicatedloop.%s\n",coin->symbol,addr->ipaddr); if ( addr->vinsfp != 0 ) fclose(addr->vinsfp); if ( addr->voutsfp != 0 ) diff --git a/iguana/iguana_ramchain.c b/iguana/iguana_ramchain.c index eaf0eb284..205ed5aba 100755 --- a/iguana/iguana_ramchain.c +++ b/iguana/iguana_ramchain.c @@ -1273,7 +1273,7 @@ int32_t iguana_ramchain_verify(struct iguana_info *coin,struct iguana_ramchain * return(0); } -int32_t iguana_ramchain_free(struct iguana_ramchain *ramchain,int32_t deleteflag) +int32_t iguana_ramchain_free(struct iguana_info *coin,struct iguana_ramchain *ramchain,int32_t deleteflag) { struct iguana_kvitem *item,*tmp; if ( ramchain->H.ROflag != 0 && ramchain->hashmem == 0 ) @@ -1326,18 +1326,7 @@ int32_t iguana_ramchain_free(struct iguana_ramchain *ramchain,int32_t deleteflag ramchain->numXspends = 0; ramchain->Xspendinds = 0; } - if ( ramchain->debitsfileptr != 0 ) - { - munmap(ramchain->debitsfileptr,ramchain->debitsfilesize); - ramchain->debitsfileptr = 0; - ramchain->debitsfilesize = 0; - } - if ( ramchain->lastspendsfileptr != 0 ) - { - munmap(ramchain->lastspendsfileptr,ramchain->lastspendsfilesize); - ramchain->lastspendsfileptr = 0; - ramchain->lastspendsfilesize = 0; - } + iguana_purgevolatiles(coin,ramchain); if ( deleteflag != 0 ) memset(ramchain,0,sizeof(*ramchain)); return(0); @@ -1345,7 +1334,7 @@ int32_t iguana_ramchain_free(struct iguana_ramchain *ramchain,int32_t deleteflag int32_t iguana_ramchain_extras(struct iguana_info *coin,struct iguana_ramchain *ramchain,struct OS_memspace *hashmem,int32_t extraflag) { - RAMCHAIN_DECLARE; int32_t iter,err=0,numhdrsi; char fname[1024]; bits256 balancehash; + RAMCHAIN_DECLARE; int32_t err=0; if ( ramchain->expanded != 0 ) { _iguana_ramchain_setptrs(RAMCHAIN_PTRS,ramchain->H.data); @@ -1357,60 +1346,7 @@ int32_t iguana_ramchain_extras(struct iguana_info *coin,struct iguana_ramchain * ramchain->A = (hashmem != 0) ? iguana_memalloc(hashmem,sizeof(struct iguana_account) * ramchain->H.data->numpkinds,1) : mycalloc('p',ramchain->H.data->numpkinds,sizeof(struct iguana_account)); ramchain->Uextras = (hashmem != 0) ? iguana_memalloc(hashmem,sizeof(*ramchain->Uextras) * ramchain->H.data->numunspents,1) : mycalloc('p',ramchain->H.data->numunspents,sizeof(*ramchain->Uextras)); } - else - { - err = -1; - for (iter=0; iter<2; iter++) - { - sprintf(fname,"DB/%s%s/accounts/debits.%d",iter==0?"ro/":"",coin->symbol,ramchain->H.data->height); - if ( (ramchain->debitsfileptr= OS_mapfile(fname,&ramchain->debitsfilesize,0)) != 0 && ramchain->debitsfilesize == sizeof(int32_t) + sizeof(bits256) + sizeof(*ramchain->A) * ramchain->H.data->numpkinds ) - { - ramchain->from_roA = (iter == 0); - numhdrsi = *(int32_t *)ramchain->debitsfileptr; - memcpy(balancehash.bytes,(void *)((long)ramchain->debitsfileptr + sizeof(numhdrsi)),sizeof(balancehash)); - if ( coin->balanceswritten == 0 ) - { - coin->balanceswritten = numhdrsi; - coin->balancehash = balancehash; - } - if ( numhdrsi == coin->balanceswritten || memcmp(balancehash.bytes,coin->balancehash.bytes,sizeof(balancehash)) == 0 ) - { - ramchain->A = (void *)((long)ramchain->debitsfileptr + sizeof(numhdrsi) + sizeof(bits256)); - sprintf(fname,"DB/%s%s/accounts/lastspends.%d",iter==0?"ro/":"",coin->symbol,ramchain->H.data->height); - if ( (ramchain->lastspendsfileptr= OS_mapfile(fname,&ramchain->lastspendsfilesize,0)) != 0 && ramchain->lastspendsfilesize == sizeof(int32_t) + sizeof(bits256) + sizeof(*ramchain->Uextras) * ramchain->H.data->numunspents ) - { - numhdrsi = *(int32_t *)ramchain->lastspendsfileptr; - memcpy(balancehash.bytes,(void *)((long)ramchain->lastspendsfileptr + sizeof(numhdrsi)),sizeof(balancehash)); - if ( numhdrsi == coin->balanceswritten || memcmp(balancehash.bytes,coin->balancehash.bytes,sizeof(balancehash)) == 0 ) - { - ramchain->Uextras = (void *)((long)ramchain->lastspendsfileptr + sizeof(numhdrsi) + sizeof(bits256)); - ramchain->from_roU = (iter == 0); - err = 0; - } else printf("ramchain map error2 balanceswritten %d vs %d hashes %x %x\n",coin->balanceswritten,numhdrsi,coin->balancehash.uints[0],balancehash.uints[0]); - } else printf("ramchain map error3 %s\n",fname); - } else printf("ramchain map error balanceswritten %d vs %d hashes %x %x\n",coin->balanceswritten,numhdrsi,coin->balancehash.uints[0],balancehash.uints[0]); - } - if ( err == 0 ) - { - //printf("mapped extra.%s\n",fname); - break; - } - ramchain->A = 0; - ramchain->Uextras = 0; - if ( ramchain->debitsfileptr != 0 ) - { - munmap(ramchain->debitsfileptr,ramchain->debitsfilesize); - ramchain->debitsfileptr = 0; - ramchain->debitsfilesize = 0; - } - if ( ramchain->lastspendsfileptr != 0 ) - { - munmap(ramchain->lastspendsfileptr,ramchain->lastspendsfilesize); - ramchain->lastspendsfileptr = 0; - ramchain->lastspendsfilesize = 0; - } - } - } + else err = iguana_mapvolatiles(coin,ramchain); } return(err); } @@ -2035,7 +1971,7 @@ long iguana_ramchain_data(struct iguana_info *coin,struct iguana_peer *addr,stru { ptr = mapchain->fileptr; fsize = mapchain->filesize; mapchain->fileptr = 0, mapchain->filesize = 0; - iguana_ramchain_free(mapchain,1); + iguana_ramchain_free(coin,mapchain,1); memset(&R,0,sizeof(R)); R.H.data = (void *)(long)((long)ptr + fpos), R.filesize = fsize; iguana_ramchain_link(&R,block->RO.hash2,block->RO.hash2,bp->hdrsi,bp->bundleheight+bundlei,bundlei,1,firsti,1); @@ -2059,7 +1995,7 @@ long iguana_ramchain_data(struct iguana_info *coin,struct iguana_peer *addr,stru bp->numspends += ramchain->H.data->numspends; bp->rawscriptspace += ramchain->H.data->scriptspace; } - iguana_ramchain_free(&R,1); + iguana_ramchain_free(coin,&R,1); } else { @@ -2077,7 +2013,7 @@ long iguana_ramchain_data(struct iguana_info *coin,struct iguana_peer *addr,stru block->fpos = -1, block->fpipbits = block->RO.recvlen = 0; //fprintf(stderr,"finished with hdrsi.%d ht.%d scripts.%u:%u\n",bp->hdrsi,bp->bundleheight,ramchain->H.scriptoffset,ramchain->H.data->scriptspace); ramchain->H.ROflag = 0; - iguana_ramchain_free(ramchain,0); + iguana_ramchain_free(coin,ramchain,0); return(fpos); } @@ -2171,7 +2107,7 @@ int32_t iguana_bundlefiles(struct iguana_info *coin,uint32_t *ipbits,void **ptrs return(num); } -void iguana_bundlemapfree(struct OS_memspace *mem,struct OS_memspace *hashmem,uint32_t *ipbits,void **ptrs,long *filesizes,int32_t num,struct iguana_ramchain *R,int32_t starti,int32_t endi) +void iguana_bundlemapfree(struct iguana_info *coin,struct OS_memspace *mem,struct OS_memspace *hashmem,uint32_t *ipbits,void **ptrs,long *filesizes,int32_t num,struct iguana_ramchain *R,int32_t starti,int32_t endi) { int32_t j,n = (endi - starti + 1); for (j=0; jH.data->lhashes[i].uints[0]); printf("%llx ht.%d bundlehashes.%s\n",(long long)mapchain->H.data->sha256.txid,mapchain->height,coin->symbol); - iguana_ramchain_free(mapchain,cmpflag); + iguana_ramchain_free(coin,mapchain,cmpflag); } iguana_mempurge(hashmem); } @@ -2373,7 +2309,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str filesizes = mycalloc('f',bp->n,sizeof(*filesizes)); if ( (num= iguana_bundlefiles(coin,ipbits,ptrs,filesizes,bp,starti,endi)) != bp->n ) { - iguana_bundlemapfree(0,0,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,0,0,ipbits,ptrs,filesizes,num,R,starti,endi); printf("iguana_bundlesaveHT: no bundlefiles error\n"); return(-1); } @@ -2408,7 +2344,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str mapchain->H.ROflag = 1; if ( fpos > filesize ) { - iguana_bundlemapfree(0,0,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,0,0,ipbits,ptrs,filesizes,num,R,starti,endi); printf("fpos error %ld > %ld mapping hdrsi.%d bundlei.%d\n",fpos,filesize,bp->hdrsi,bundlei); break; } @@ -2454,7 +2390,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str block->RO.recvlen = 0; bp->issued[bundlei] = 0; } - iguana_bundlemapfree(0,0,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,0,0,ipbits,ptrs,filesizes,num,R,starti,endi); printf("error mapping hdrsi.%d bundlei.%d\n",bp->hdrsi,bundlei); return(-1); } @@ -2468,7 +2404,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str if ( iguana_ramchain_alloc(coin,dest,mem,&HASHMEM,numtxids,numunspents,numspends,numpkinds,numexternaltxids,scriptspace+sigspace,bp->bundleheight+starti,bp_n) < 0 ) { printf("error iguana_ramchain_alloc for bundleheight.%d\n",bp->bundleheight); - iguana_bundlemapfree(mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); return(-1); } iguana_ramchain_link(dest,bp->hashes[starti],bp->hashes[endi],bp->hdrsi,bp->bundleheight,0,bp->n,firsti,0); @@ -2489,7 +2425,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str block->fpipbits = 0; bp->issued[i] = 0; block->issued = 0; - iguana_bundlemapfree(mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); return(-1); } //destB[i] = block->RO; @@ -2538,7 +2474,7 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str //char str[65]; printf("d.%d ht.%d %s saved lag.%d elapsed.%ld\n",depth,dest->height,mbstr(str,dest->H.data->allocsize),now-starttime,time(NULL)-now); retval = 0; } else bp->generrs++; - iguana_bundlemapfree(mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); + iguana_bundlemapfree(coin,mem,&HASHMEM,ipbits,ptrs,filesizes,num,R,starti,endi); if ( retval == 0 )//|| bp->generrs > 3 ) { char dirname[1024]; @@ -2556,18 +2492,18 @@ int32_t iguana_bundlesaveHT(struct iguana_info *coin,struct OS_memspace *mem,str iguana_bundleload(coin,&newchain,bp,0); newchain.A = 0; } - iguana_ramchain_free(dest,0); + iguana_ramchain_free(coin,dest,0); bp->ramchain = newchain; //printf("finished bundlesave.%d\n",bp->bundleheight); return(retval); } -void iguana_mergefree(struct OS_memspace *mem,struct iguana_ramchain *A,struct iguana_ramchain *B,struct OS_memspace *hashmem,struct OS_memspace *hashmemA,struct OS_memspace *hashmemB) +void iguana_mergefree(struct iguana_info *coin,struct OS_memspace *mem,struct iguana_ramchain *A,struct iguana_ramchain *B,struct OS_memspace *hashmem,struct OS_memspace *hashmemA,struct OS_memspace *hashmemB) { if ( A != 0 ) - iguana_ramchain_free(A,0); + iguana_ramchain_free(coin,A,0); if ( B != 0 ) - iguana_ramchain_free(B,0); + iguana_ramchain_free(coin,B,0); if ( mem != 0 ) iguana_mempurge(mem); if ( hashmemA != 0 ) @@ -2600,7 +2536,7 @@ int32_t iguana_bundlemergeHT(struct iguana_info *coin,struct OS_memspace *mem,st if ( A == 0 || B == 0 || A->H.data == 0 || B->H.data == 0 || (A->H.data->allocsize + B->H.data->allocsize) > IGUANA_MAXRAMCHAINSIZE ) { printf("MERGE error %d[%d] %d[%d]\n",A->height,A->numblocks,B->height,B->numblocks); - iguana_mergefree(mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); + iguana_mergefree(coin,mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); return(-1); } if ( A->H.data != 0 && B->H.data != 0 && B->height == A->height+A->numblocks ) @@ -2608,7 +2544,7 @@ int32_t iguana_bundlemergeHT(struct iguana_info *coin,struct OS_memspace *mem,st if ( iguana_ramchain_alloc(coin,dest,mem,&HASHMEM,(A->H.data->numtxids+B->H.data->numtxids),(A->H.data->numunspents+B->H.data->numunspents),(A->H.data->numspends+B->H.data->numspends),(A->H.data->numpkinds+B->H.data->numpkinds),(A->H.data->numexternaltxids+B->H.data->numexternaltxids),A->H.data->scriptspace,A->height,A->numblocks + B->numblocks) < 0 ) { printf("depth.%d ht.%d fsize.%s ERROR alloc lag.%d elapsed.%ld\n",depth,dest->height,mbstr(str,dest->H.data->allocsize),now-starttime,time(NULL)-now); - iguana_mergefree(mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); + iguana_mergefree(coin,mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); return(-1); } depth++; @@ -2625,7 +2561,7 @@ int32_t iguana_bundlemergeHT(struct iguana_info *coin,struct OS_memspace *mem,st { printf("merging isnt setup to save the blockROs\n"); printf("depth.%d ht.%d fsize.%s MERGED %d[%d] and %d[%d] lag.%d elapsed.%ld bp.%d -> %d\n",depth,dest->height,mbstr(str,dest->H.data->allocsize),A->height,A->numblocks,B->height,B->numblocks,now-starttime,time(NULL)-now,bp->bundleheight,nextbp->bundleheight); - iguana_mergefree(mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); + iguana_mergefree(coin,mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); bp->mergefinish = 0; nextbp->mergefinish = (uint32_t)time(NULL); bp->nextbp = nextbp->nextbp; @@ -2638,9 +2574,9 @@ int32_t iguana_bundlemergeHT(struct iguana_info *coin,struct OS_memspace *mem,st else { bp->mergefinish = nextbp->mergefinish = 0; - iguana_mergefree(mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); + iguana_mergefree(coin,mem,A,B,&HASHMEM,&HASHMEMA,&HASHMEMB); } - iguana_ramchain_free(dest,0); + iguana_ramchain_free(coin,dest,0); depth--; } else printf("error merging A.%d [%d] and B.%d [%d]\n",A->height,A->numblocks,B->height,B->numblocks); coin->merging--; diff --git a/iguana/iguana_unspents.c b/iguana/iguana_unspents.c index 3daa5eb94..1dbd12517 100755 --- a/iguana/iguana_unspents.c +++ b/iguana/iguana_unspents.c @@ -274,7 +274,7 @@ int32_t iguana_utxogen(struct iguana_info *coin,struct iguana_bundle *bp) { static uint64_t total,emitted; int32_t spendind,height,n,numtxid,errs=0,emit=0; uint32_t unspentind; struct iguana_bundle *spentbp; - FILE *fp; char fname[1024],str[65]; int32_t retval = -1; + FILE *fp; char fname[1024],str[65]; uint32_t now; int32_t retval = -1; bits256 prevhash,zero,sha256; struct iguana_unspent *u; long fsize; struct iguana_txid *nextT; struct iguana_spend *S,*s; struct iguana_spendvector *ptr; struct iguana_ramchain *ramchain; ramchain = &bp->ramchain; @@ -292,6 +292,7 @@ int32_t iguana_utxogen(struct iguana_info *coin,struct iguana_bundle *bp) ptr = mycalloc('x',sizeof(*ptr),n); total += n; height = bp->bundleheight; + now = (uint32_t)time(NULL); //printf("start UTXOGEN.%d max.%d ptr.%p\n",bp->bundleheight,n,ptr); for (spendind=ramchain->H.data->firsti; spendindhdrsi,s)) != 0 ) { + if ( now > spentbp->lastprefetch+10 || (spentbp->dirty % 100000) == 0 ) + { + //printf("current.%d prefetch.[%d]\n",spentbp == bp,spentbp->hdrsi); + iguana_ramchain_prefetch(coin,&spentbp->ramchain); + bp->lastprefetch = now; + } + spentbp->dirty++; if ( (ptr[emit].ind= unspentind) != 0 && spentbp->hdrsi < bp->hdrsi ) { ptr[emit].hdrsi = spentbp->hdrsi; @@ -369,7 +377,7 @@ int32_t iguana_utxogen(struct iguana_info *coin,struct iguana_bundle *bp) } if ( ptr != 0 ) myfree(ptr,sizeof(*ptr) * n); - printf("utxo %d spendinds.[%d] errs.%d [%.2f%%] emitted.%d %s of %d | ",spendind,bp->hdrsi,errs,100.*(double)emitted/(total+1),emit,mbstr(str,sizeof(*ptr) * emit),n); + printf("utxo %d spendinds.[%d] errs.%d [%.2f%%] emitted.%d %s of %d\n",spendind,bp->hdrsi,errs,100.*(double)emitted/(total+1),emit,mbstr(str,sizeof(*ptr) * emit),n); if ( errs != 0 ) exit(-1); return(-errs); @@ -455,14 +463,14 @@ int32_t iguana_balancegen(struct iguana_info *coin,struct iguana_bundle *bp,int3 now = (uint32_t)time(NULL); if ( spentbp != 0 && unspentind > 0 && unspentind < spentbp->ramchain.H.data->numunspents ) { - if ( now > bp->lastprefetch+30 ) + if ( now > spentbp->lastprefetch+10 || (spentbp->dirty % 100000) == 0 ) { //printf("current.%d prefetch.[%d]\n",spentbp == bp,spentbp->hdrsi); iguana_ramchain_prefetch(coin,&spentbp->ramchain); bp->lastprefetch = now; } spentbp->dirty++; - //if ( incremental == 0 ) + if ( incremental == 0 ) { if ( spentbp->ramchain.Uextras == 0 ) { @@ -549,6 +557,243 @@ int32_t iguana_balancegen(struct iguana_info *coin,struct iguana_bundle *bp,int3 return(-errs); } + +void iguana_purgevolatiles(struct iguana_info *coin,struct iguana_ramchain *ramchain) +{ + ramchain->A = 0; + ramchain->Uextras = 0; + if ( ramchain->debitsfileptr != 0 ) + { + munmap(ramchain->debitsfileptr,ramchain->debitsfilesize); + ramchain->debitsfileptr = 0; + ramchain->debitsfilesize = 0; + } + if ( ramchain->lastspendsfileptr != 0 ) + { + munmap(ramchain->lastspendsfileptr,ramchain->lastspendsfilesize); + ramchain->lastspendsfileptr = 0; + ramchain->lastspendsfilesize = 0; + } +} + +int32_t iguana_mapvolatiles(struct iguana_info *coin,struct iguana_ramchain *ramchain) +{ + int32_t iter,numhdrsi,err = -1; char fname[1024]; bits256 balancehash; + for (iter=0; iter<2; iter++) + { + sprintf(fname,"DB/%s%s/accounts/debits.%d",iter==0?"ro/":"",coin->symbol,ramchain->H.data->height); + if ( (ramchain->debitsfileptr= OS_mapfile(fname,&ramchain->debitsfilesize,0)) != 0 && ramchain->debitsfilesize == sizeof(int32_t) + sizeof(bits256) + sizeof(*ramchain->A) * ramchain->H.data->numpkinds ) + { + ramchain->from_roA = (iter == 0); + numhdrsi = *(int32_t *)ramchain->debitsfileptr; + memcpy(balancehash.bytes,(void *)((long)ramchain->debitsfileptr + sizeof(numhdrsi)),sizeof(balancehash)); + if ( coin->balanceswritten == 0 ) + { + coin->balanceswritten = numhdrsi; + coin->balancehash = balancehash; + } + if ( numhdrsi == coin->balanceswritten || memcmp(balancehash.bytes,coin->balancehash.bytes,sizeof(balancehash)) == 0 ) + { + ramchain->A = (void *)((long)ramchain->debitsfileptr + sizeof(numhdrsi) + sizeof(bits256)); + sprintf(fname,"DB/%s%s/accounts/lastspends.%d",iter==0?"ro/":"",coin->symbol,ramchain->H.data->height); + if ( (ramchain->lastspendsfileptr= OS_mapfile(fname,&ramchain->lastspendsfilesize,0)) != 0 && ramchain->lastspendsfilesize == sizeof(int32_t) + sizeof(bits256) + sizeof(*ramchain->Uextras) * ramchain->H.data->numunspents ) + { + numhdrsi = *(int32_t *)ramchain->lastspendsfileptr; + memcpy(balancehash.bytes,(void *)((long)ramchain->lastspendsfileptr + sizeof(numhdrsi)),sizeof(balancehash)); + if ( numhdrsi == coin->balanceswritten || memcmp(balancehash.bytes,coin->balancehash.bytes,sizeof(balancehash)) == 0 ) + { + ramchain->Uextras = (void *)((long)ramchain->lastspendsfileptr + sizeof(numhdrsi) + sizeof(bits256)); + ramchain->from_roU = (iter == 0); + err = 0; + } else printf("ramchain map error2 balanceswritten %d vs %d hashes %x %x\n",coin->balanceswritten,numhdrsi,coin->balancehash.uints[0],balancehash.uints[0]); + } else printf("ramchain map error3 %s\n",fname); + } else printf("ramchain map error balanceswritten %d vs %d hashes %x %x\n",coin->balanceswritten,numhdrsi,coin->balancehash.uints[0],balancehash.uints[0]); + } + if ( err == 0 ) + { + //printf("mapped extra.%s\n",fname); + break; + } + iguana_purgevolatiles(coin,ramchain); + } + return(err); +} + +void iguana_allocvolatile(struct iguana_info *coin,struct iguana_ramchain *ramchain) +{ + if ( ramchain != 0 && ramchain->H.data != 0 ) + { + ramchain->A = calloc(sizeof(*ramchain->A),ramchain->H.data->numpkinds + 16); + ramchain->Uextras = calloc(sizeof(*ramchain->Uextras),ramchain->H.data->numunspents + 16); + if ( ramchain->debitsfileptr != 0 ) + { + memcpy(ramchain->A,(void *)((long)ramchain->debitsfileptr + sizeof(int32_t) + sizeof(bits256)),sizeof(*ramchain->A) * ramchain->H.data->numpkinds); + munmap(ramchain->debitsfileptr,ramchain->debitsfilesize); + ramchain->debitsfileptr = 0; + ramchain->debitsfilesize = 0; + } + if ( ramchain->lastspendsfileptr != 0 ) + { + memcpy(ramchain->Uextras,(void *)((long)ramchain->lastspendsfileptr + sizeof(int32_t) + sizeof(bits256)),sizeof(*ramchain->Uextras) * ramchain->H.data->numunspents); + munmap(ramchain->lastspendsfileptr,ramchain->lastspendsfilesize); + ramchain->lastspendsfileptr = 0; + ramchain->lastspendsfilesize = 0; + } + } else printf("illegal ramchain.%p\n",ramchain); +} + +int32_t iguana_balanceflush(struct iguana_info *coin,int32_t refhdrsi,int32_t purgedist) +{ + int32_t hdrsi,numpkinds,iter,numhdrsi,numunspents,err; struct iguana_bundle *bp; + char fname[1024],fname2[1024],destfname[1024]; bits256 balancehash; FILE *fp,*fp2; + struct iguana_utxo *Uptr; struct iguana_account *Aptr; struct sha256_vstate vstate; + vupdate_sha256(balancehash.bytes,&vstate,0,0); + for (hdrsi=0; hdrsibundlescount; hdrsi++) + if ( (bp= coin->bundles[hdrsi]) == 0 || bp->balancefinish <= 1 || bp->ramchain.H.data == 0 || bp->ramchain.A == 0 || bp->ramchain.Uextras == 0 ) + break; + if ( hdrsi <= coin->balanceswritten || hdrsi < refhdrsi ) + return(0); + numhdrsi = hdrsi; + vupdate_sha256(balancehash.bytes,&vstate,0,0); + for (iter=0; iter<3; iter++) + { + for (hdrsi=0; hdrsibundles[hdrsi]) != 0 && bp->ramchain.H.data != 0 && (numpkinds= bp->ramchain.H.data->numpkinds) > 0 && (numunspents= bp->ramchain.H.data->numunspents) > 0 && (Aptr= bp->ramchain.A) != 0 && (Uptr= bp->ramchain.Uextras) != 0 ) + { + sprintf(fname,"accounts/%s/debits.%d",coin->symbol,bp->bundleheight); + sprintf(fname2,"accounts/%s/lastspends.%d",coin->symbol,bp->bundleheight); + if ( iter == 0 ) + { + vupdate_sha256(balancehash.bytes,&vstate,(void *)Aptr,sizeof(*Aptr)*numpkinds); + vupdate_sha256(balancehash.bytes,&vstate,(void *)Uptr,sizeof(*Uptr)*numunspents); + } + else if ( iter == 1 ) + { + if ( (fp= fopen(fname,"wb")) != 0 && (fp2= fopen(fname2,"wb")) != 0 ) + { + err = -1; + if ( fwrite(&numhdrsi,1,sizeof(numhdrsi),fp) == sizeof(numhdrsi) && fwrite(&numhdrsi,1,sizeof(numhdrsi),fp2) == sizeof(numhdrsi) && fwrite(balancehash.bytes,1,sizeof(balancehash),fp) == sizeof(balancehash) && fwrite(balancehash.bytes,1,sizeof(balancehash),fp2) == sizeof(balancehash) ) + { + if ( fwrite(Aptr,sizeof(*Aptr),numpkinds,fp) == numpkinds ) + { + if ( fwrite(Uptr,sizeof(*Uptr),numunspents,fp2) == numunspents ) + { + //bp->dirty = 0; + err = 0; + free(bp->ramchain.A), bp->ramchain.A = 0; + free(bp->ramchain.Uextras), bp->ramchain.Uextras = 0; + printf("[%d] of %d saved (%s) and (%s)\n",hdrsi,numhdrsi,fname,fname2); + } + } + } + if ( err != 0 ) + { + printf("balanceflush.%s error iter.%d hdrsi.%d\n",coin->symbol,iter,hdrsi); + fclose(fp); + fclose(fp2); + return(-1); + } + fclose(fp), fclose(fp2); + } + else + { + printf("error opening %s or %s %p\n",fname,fname2,fp); + if ( fp != 0 ) + fclose(fp); + } + } + else if ( iter == 2 ) + { + sprintf(destfname,"DB/%s/accounts/debits.%d",coin->symbol,bp->bundleheight); + if ( OS_copyfile(fname,destfname,1) < 0 ) + { + printf("balances error copying (%s) -> (%s)\n",fname,destfname); + return(-1); + } + sprintf(destfname,"DB/%s/accounts/lastspends.%d",coin->symbol,bp->bundleheight); + if ( OS_copyfile(fname2,destfname,1) < 0 ) + { + printf("balances error copying (%s) -> (%s)\n",fname2,destfname); + return(-1); + } + printf("%s %s\n",fname,destfname); + /*if ( hdrsi > numhdrsi-purgedist && numhdrsi >= purgedist ) + { + sprintf(destfname,"DB/%s/accounts/debits_%d.%d",coin->symbol,numhdrsi-purgedist,bp->bundleheight); + OS_removefile(destfname,0); + sprintf(destfname,"DB/%s/accounts/lastspends_%d.%d",coin->symbol,numhdrsi-purgedist,bp->bundleheight); + OS_removefile(destfname,0); + }*/ + } + } + else + { + printf("error loading [%d] Aptr.%p Uptr.%p numpkinds.%u numunspents.%u\n",hdrsi,Aptr,Uptr,numpkinds,numunspents); + return(-1); + } + } + } + coin->balancehash = balancehash; + coin->balanceswritten = numhdrsi; + for (hdrsi=0; hdrsibundles[hdrsi]) == 0 || iguana_mapvolatiles(coin,&bp->ramchain) != 0 ) + printf("error mapping bundle.[%d]\n",hdrsi); + char str[65]; printf("BALANCES WRITTEN for %d bundles %s\n",coin->balanceswritten,bits256_str(str,coin->balancehash)); + return(coin->balanceswritten); +} + +int32_t iguana_balancecalc(struct iguana_info *coin,struct iguana_bundle *bp,int32_t incremental) +{ + uint32_t starttime,j,flag = 0; struct iguana_bundle *prevbp; + if ( bp->balancefinish > 1 ) + { + printf("make sure DB files have this bp.%d\n",bp->hdrsi); + iguana_validateQ(coin,bp); + return(flag); + } + if ( bp != 0 && coin != 0 && (bp->hdrsi == 0 || (prevbp= coin->bundles[bp->hdrsi-1]) != 0) ) + { + for (j=0; jhdrsi; j++) + { + if ( (prevbp= coin->bundles[j]) == 0 || prevbp->utxofinish <= 1 || prevbp->balancefinish == 0 ) + break; + } + if ( bp->utxofinish > 1 && bp->balancefinish <= 1 && bp->hdrsi == j ) + { + starttime = (uint32_t)time(NULL); + for (j=0; j<=bp->hdrsi; j++) + iguana_allocvolatile(coin,&coin->bundles[j]->ramchain); + if ( iguana_balancegen(coin,bp,incremental) < 0 ) + { + printf("GENERATE BALANCES ERROR ht.%d\n",bp->bundleheight); + exit(-1); + } + bp->balancefinish = (uint32_t)time(NULL); + printf("GENERATED BALANCES for ht.%d duration %d seconds\n",bp->bundleheight,bp->balancefinish - (uint32_t)starttime); + bp->queued = 0; + iguana_validateQ(coin,bp); + if ( bp->hdrsi >= coin->longestchain/coin->chain->bundlesize-1 && bp->hdrsi >= coin->balanceswritten ) + { + iguana_balanceflush(coin,bp->hdrsi,3); + printf("balanceswritten.%d flushed bp->hdrsi %d vs %d coin->longestchain/coin->chain->bundlesize\n",coin->balanceswritten,bp->hdrsi,coin->longestchain/coin->chain->bundlesize); + } + flag++; + } + else + { + //printf("third case.%d utxo.%u balance.%u prev.%u\n",bp->hdrsi,bp->utxofinish,bp->balancefinish,prevbp!=0?prevbp->utxofinish:-1); + coin->pendbalances--; + iguana_balancesQ(coin,bp); + } + } + return(flag); +} + int32_t iguana_bundlevalidate(struct iguana_info *coin,struct iguana_bundle *bp) { if ( bp->validated <= 1 ) diff --git a/iguana/main.c b/iguana/main.c index 4f16b75b8..5d4ca577f 100755 --- a/iguana/main.c +++ b/iguana/main.c @@ -341,104 +341,10 @@ mksquashfs DB/BTC BTC.squash1M -b 1048576 sudo mount BTC.xz DB/ro/BTC -t squashfs -o loop */ -int32_t iguana_balanceflush(struct iguana_info *coin,int32_t refhdrsi,int32_t purgedist) -{ - int32_t hdrsi,numpkinds,iter,numhdrsi,numunspents,err; struct iguana_bundle *bp; - char fname[1024],fname2[1024],destfname[1024]; bits256 balancehash; FILE *fp,*fp2; - struct iguana_utxo *Uptr; struct iguana_account *Aptr; struct sha256_vstate vstate; - vupdate_sha256(balancehash.bytes,&vstate,0,0); - for (hdrsi=0; hdrsibundlescount; hdrsi++) - if ( (bp= coin->bundles[hdrsi]) == 0 || bp->balancefinish <= 1 || bp->ramchain.H.data == 0 || bp->ramchain.A == 0 || bp->ramchain.Uextras == 0 ) - break; - if ( hdrsi <= coin->balanceswritten || hdrsi < refhdrsi ) - return(0); - numhdrsi = hdrsi; - vupdate_sha256(balancehash.bytes,&vstate,0,0); - for (iter=0; iter<3; iter++) - { - for (hdrsi=0; hdrsibundles[hdrsi]) != 0 && bp->ramchain.H.data != 0 && (numpkinds= bp->ramchain.H.data->numpkinds) > 0 && (numunspents= bp->ramchain.H.data->numunspents) > 0 && (Aptr= bp->ramchain.A) != 0 && (Uptr= bp->ramchain.Uextras) != 0 ) - { - sprintf(fname,"accounts/%s/debits.%d",coin->symbol,bp->bundleheight); - sprintf(fname2,"accounts/%s/lastspends.%d",coin->symbol,bp->bundleheight); - if ( iter == 0 ) - { - vupdate_sha256(balancehash.bytes,&vstate,(void *)Aptr,sizeof(*Aptr)*numpkinds); - vupdate_sha256(balancehash.bytes,&vstate,(void *)Uptr,sizeof(*Uptr)*numunspents); - } - else if ( iter == 1 ) - { - if ( (fp= fopen(fname,"wb")) != 0 && (fp2= fopen(fname2,"wb")) != 0 ) - { - err = -1; - if ( fwrite(&numhdrsi,1,sizeof(numhdrsi),fp) == sizeof(numhdrsi) && fwrite(&numhdrsi,1,sizeof(numhdrsi),fp2) == sizeof(numhdrsi) && fwrite(balancehash.bytes,1,sizeof(balancehash),fp) == sizeof(balancehash) && fwrite(balancehash.bytes,1,sizeof(balancehash),fp2) == sizeof(balancehash) ) - { - if ( fwrite(Aptr,sizeof(*Aptr),numpkinds,fp) == numpkinds ) - { - if ( fwrite(Uptr,sizeof(*Uptr),numunspents,fp2) == numunspents ) - { - //bp->dirty = 0; - err = 0; - printf("[%d] of %d saved (%s) and (%s)\n",hdrsi,numhdrsi,fname,fname2); - } - } - } - if ( err != 0 ) - { - printf("balanceflush.%s error iter.%d hdrsi.%d\n",coin->symbol,iter,hdrsi); - fclose(fp); - fclose(fp2); - return(-1); - } - fclose(fp), fclose(fp2); - } - else - { - printf("error opening %s or %s %p\n",fname,fname2,fp); - if ( fp != 0 ) - fclose(fp); - } - } - else if ( iter == 2 ) - { - sprintf(destfname,"DB/%s/accounts/debits.%d",coin->symbol,bp->bundleheight); - if ( OS_copyfile(fname,destfname,1) < 0 ) - { - printf("balances error copying (%s) -> (%s)\n",fname,destfname); - return(-1); - } - sprintf(destfname,"DB/%s/accounts/lastspends.%d",coin->symbol,bp->bundleheight); - if ( OS_copyfile(fname2,destfname,1) < 0 ) - { - printf("balances error copying (%s) -> (%s)\n",fname2,destfname); - return(-1); - } - printf("%s %s\n",fname,destfname); - /*if ( hdrsi > numhdrsi-purgedist && numhdrsi >= purgedist ) - { - sprintf(destfname,"DB/%s/accounts/debits_%d.%d",coin->symbol,numhdrsi-purgedist,bp->bundleheight); - OS_removefile(destfname,0); - sprintf(destfname,"DB/%s/accounts/lastspends_%d.%d",coin->symbol,numhdrsi-purgedist,bp->bundleheight); - OS_removefile(destfname,0); - }*/ - } - } else printf("error loading [%d] Aptr.%p Uptr.%p numpkinds.%u numunspents.%u\n",hdrsi,Aptr,Uptr,numpkinds,numunspents); - } - } - coin->balancehash = balancehash; - coin->balanceswritten = numhdrsi; - char str[65]; printf("BALANCES WRITTEN for %d bundles %s\n",coin->balanceswritten,bits256_str(str,coin->balancehash)); - return(coin->balanceswritten); -} void mainloop(struct supernet_info *myinfo) { - int32_t i,j,flag; char str[2048]; struct iguana_info *coin; struct iguana_helper *ptr; struct iguana_bundle *bp,*prevbp = 0; + int32_t i,flag; struct iguana_info *coin; struct iguana_helper *ptr; struct iguana_bundle *bp; sleep(3); printf("mainloop\n"); while ( 1 ) @@ -449,45 +355,10 @@ void mainloop(struct supernet_info *myinfo) for (i=0; iactive != 0 && (bp= coin->current) != 0 && coin->started != 0 ) { - iguana_bundlestats(coin,str); iguana_realtime_update(coin); if ( (ptr= queue_dequeue(&balancesQ,0)) != 0 ) { - if ( (bp= ptr->bp) != 0 && ptr->coin != 0 && (bp->hdrsi == 0 || (prevbp= coin->bundles[bp->hdrsi-1]) != 0) ) - { - for (j=0; jhdrsi; j++) - { - if ( (prevbp= coin->bundles[j]) == 0 || prevbp->utxofinish <= 1 || prevbp->balancefinish == 0 ) - break; - } - if ( bp->utxofinish > 1 && bp->balancefinish <= 1 && bp->hdrsi == j ) - { - if ( bp->ramchain.Uextras == 0 ) - { - printf("alloc Uextras.[%d]\n",bp->hdrsi); - bp->ramchain.Uextras = calloc(sizeof(*bp->ramchain.Uextras),bp->ramchain.H.data->numunspents + 16); - } - if ( bp->ramchain.A == 0 ) - { - printf("alloc A2.[%d]\n",bp->hdrsi); - bp->ramchain.A = calloc(sizeof(*bp->ramchain.A),bp->ramchain.H.data->numpkinds + 16); - } - iguana_balancecalc(ptr->coin,bp,0); - bp->queued = 0; - if ( bp->hdrsi == coin->longestchain/coin->chain->bundlesize-1 ) - { - flag++; - iguana_balanceflush(ptr->coin,bp->hdrsi,3); - printf("flushed bp->hdrsi %d vs %d coin->longestchain/coin->chain->bundlesize\n",bp->hdrsi,coin->longestchain/coin->chain->bundlesize); - } - } - else - { - //printf("third case.%d utxo.%u balance.%u prev.%u\n",bp->hdrsi,bp->utxofinish,bp->balancefinish,prevbp!=0?prevbp->utxofinish:-1); - coin->pendbalances--; - iguana_balancesQ(coin,bp); - } - } + iguana_balancecalc(ptr->coin,ptr->bp,0); myfree(ptr,ptr->allocsize); } }