From b51f01ed100f97076584c6900f73deb5dc5f6bbc Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Wed, 29 Apr 2015 19:17:07 +0200 Subject: [PATCH 01/21] resources scripts --- CMakeLists.txt | 3 +++ cmake/EthUtils.cmake | 28 ++++++++++++++++++++++++++ cmake/scripts/resource.cpp.in | 26 ++++++++++++++++++++++++ cmake/scripts/resources.cmake | 37 +++++++++++++++++++++++++++++++++++ cmake/scripts/test.cmake | 6 ++++++ 5 files changed, 100 insertions(+) create mode 100644 cmake/scripts/resource.cpp.in create mode 100644 cmake/scripts/resources.cmake create mode 100644 cmake/scripts/test.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index ff8732156..a770e9704 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -383,6 +383,9 @@ if (GUI) endif() +set(ETH_RESOURCES "LICENSE" "README.md") +eth_create_resources(ETH_RESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/here.h") + #unset(TARGET_PLATFORM CACHE) if (WIN32) diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake index 69690156a..5d271f341 100644 --- a/cmake/EthUtils.cmake +++ b/cmake/EthUtils.cmake @@ -62,3 +62,31 @@ macro(eth_add_test NAME) endmacro() +# Based on +# http://stackoverflow.com/questions/11813271/embed-resources-eg-shader-code-images-into-executable-library-with-cmake +# Creates C resources file from files +function(eth_create_resources bins output) + set(tmp_output "${output}.tmp") + # Create empty output file + file(WRITE ${tmp_output} "") + # Collect input files +# file(GLOB bins ${dir}/*) + # Iterate through input files + foreach(bin ${${bins}}) + # Get short filename + string(REGEX MATCH "([^/]+)$" filename ${bin}) + # Replace filename spaces & extension separator for C compatibility + string(REGEX REPLACE "\\.| " "_" filename ${filename}) + # Add eth prefix (qt does the same thing) + set(filename "eth_${filename}") + # full name + file(GLOB the_name ${bin}) + # Read hex data from file + file(READ ${bin} filedata HEX) + # Convert hex data for C compatibility + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) + # Append data to output file + file(APPEND ${tmp_output} "static const unsigned char ${filename}[] = {\n // ${the_name}\n ${filedata}};\nstatic const unsigned ${filename}_size = sizeof(${filename});\n") + endforeach() + replace_if_different("${tmp_output}" "${output}") +endfunction() diff --git a/cmake/scripts/resource.cpp.in b/cmake/scripts/resource.cpp.in new file mode 100644 index 000000000..369f7b56f --- /dev/null +++ b/cmake/scripts/resource.cpp.in @@ -0,0 +1,26 @@ + +#include +#include +#include +#include + +using namespace std; + +${ETH_RESULT_DATA} + +static unordered_map eth_resources; +static unordered_map eth_sizes; + +void initResources() +{ + ${ETH_RESULT_INIT} + //eth_resources["LICENSE"] = (char const*)eth_LICENSE; + //eth_sizes["LICENSE"] = sizeof(eth_LICENSE); +} + +string loadResource(string _name) +{ + ostringstream bistream; + bistream.write(eth_resources[_name], eth_sizes[_name]); + return bistream.str(); +} diff --git a/cmake/scripts/resources.cmake b/cmake/scripts/resources.cmake new file mode 100644 index 000000000..2c92103ff --- /dev/null +++ b/cmake/scripts/resources.cmake @@ -0,0 +1,37 @@ + +# cmake -DETH_RES_FILE=... -DETH_DST_NAME=... -P scripts/resources.cmake +# cmake -DETH_RES_FILE=test.cmake -DETH_DST_NAME=dst -P resources.cmake + +# should define ETH_RESOURCES +include(${ETH_RES_FILE}) + +set(ETH_RESULT_DATA) +set(ETH_RESULT_INIT) + +# resource is a name visible for cpp application +foreach(resource ${ETH_RESOURCES}) + + # filename is the name of file which will be used in app + set(filename ${${resource}}) + + # filedata is a file content + file(READ ${filename} filedata HEX) + + # Convert hex data for C compatibility + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) + + # append static variables to result variable + list(APPEND ${ETH_RESULT_DATA} "static const unsigned char eth_${resource}[] = {\n // ${filename}\n ${filedata}\n};\n") + + # append init resources + list(APPEND ${ETH_RESULT_INIT} " eth_resources[\"${resource}\"] = (char const*)eth_${resource};\n") + list(APPEND ${ETH_RESULT_INIT} " eth_sizes[\"${resource}\"] = sizeof(eth_${resource});\n") + +endforeach(resource) + +configure_file("resource.cpp.in" "${ETH_DST_NAME}.cpp.tmp") + +include("../EthUtils.cmake") +replace_if_different("${ETH_DST_NAME}.cpp.tmp" "${ETH_DST_NAME}.cpp") +replace_if_different("resource.h" "${ETH_DST_NAME}.h") + diff --git a/cmake/scripts/test.cmake b/cmake/scripts/test.cmake new file mode 100644 index 000000000..883ed324e --- /dev/null +++ b/cmake/scripts/test.cmake @@ -0,0 +1,6 @@ + +set(copydlls "copydlls.cmake") +set(conf "configure.cmake") + +set(ETH_RESOURCES "copyddls" "conf") + From a7996f4de717da9a82fb740e8d7abbb64d90c1d7 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Wed, 29 Apr 2015 22:38:58 +0200 Subject: [PATCH 02/21] finished resource scripts --- CMakeLists.txt | 3 --- cmake/EthUtils.cmake | 30 +++------------------- cmake/scripts/resource.cpp.in | 21 +++++++--------- cmake/scripts/resource.h.in | 26 +++++++++++++++++++ cmake/scripts/resources.cmake | 47 ++++++++++++++++++++++++++--------- cmake/scripts/test.cmake | 6 ----- 6 files changed, 74 insertions(+), 59 deletions(-) create mode 100644 cmake/scripts/resource.h.in delete mode 100644 cmake/scripts/test.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index a770e9704..ff8732156 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -383,9 +383,6 @@ if (GUI) endif() -set(ETH_RESOURCES "LICENSE" "README.md") -eth_create_resources(ETH_RESOURCES "${CMAKE_CURRENT_SOURCE_DIR}/here.h") - #unset(TARGET_PLATFORM CACHE) if (WIN32) diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake index 5d271f341..147ce0737 100644 --- a/cmake/EthUtils.cmake +++ b/cmake/EthUtils.cmake @@ -62,31 +62,9 @@ macro(eth_add_test NAME) endmacro() -# Based on -# http://stackoverflow.com/questions/11813271/embed-resources-eg-shader-code-images-into-executable-library-with-cmake # Creates C resources file from files -function(eth_create_resources bins output) - set(tmp_output "${output}.tmp") - # Create empty output file - file(WRITE ${tmp_output} "") - # Collect input files -# file(GLOB bins ${dir}/*) - # Iterate through input files - foreach(bin ${${bins}}) - # Get short filename - string(REGEX MATCH "([^/]+)$" filename ${bin}) - # Replace filename spaces & extension separator for C compatibility - string(REGEX REPLACE "\\.| " "_" filename ${filename}) - # Add eth prefix (qt does the same thing) - set(filename "eth_${filename}") - # full name - file(GLOB the_name ${bin}) - # Read hex data from file - file(READ ${bin} filedata HEX) - # Convert hex data for C compatibility - string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) - # Append data to output file - file(APPEND ${tmp_output} "static const unsigned char ${filename}[] = {\n // ${the_name}\n ${filedata}};\nstatic const unsigned ${filename}_size = sizeof(${filename});\n") - endforeach() - replace_if_different("${tmp_output}" "${output}") +function(eth_add_resources TARGET RESOURCE_FILE) + add_custom_command(TARGET ${TARGET} PRE_BUILD + COMMAND ${CMAKE_COMMAND} -DETH_RES_FILE="${RESOURCE_FILE}" -P "${ETH_SCRIPTS_DIR}/resources.cmake" + ) endfunction() diff --git a/cmake/scripts/resource.cpp.in b/cmake/scripts/resource.cpp.in index 369f7b56f..b73a8df1a 100644 --- a/cmake/scripts/resource.cpp.in +++ b/cmake/scripts/resource.cpp.in @@ -1,26 +1,23 @@ +// this file is autogenerated, do not modify!!! -#include #include #include -#include +#include "${ETH_RESOURCE_NAME}.h" using namespace std; +using namespace dev; +using namespace dev::eth; -${ETH_RESULT_DATA} - -static unordered_map eth_resources; -static unordered_map eth_sizes; - -void initResources() +${ETH_RESOURCE_NAME}::${ETH_RESOURCE_NAME}() { - ${ETH_RESULT_INIT} - //eth_resources["LICENSE"] = (char const*)eth_LICENSE; - //eth_sizes["LICENSE"] = sizeof(eth_LICENSE); +${ETH_RESULT_DATA} +${ETH_RESULT_INIT} } -string loadResource(string _name) +string ${ETH_RESOURCE_NAME}::loadResourceAsString(string _name) { ostringstream bistream; bistream.write(eth_resources[_name], eth_sizes[_name]); return bistream.str(); } + diff --git a/cmake/scripts/resource.h.in b/cmake/scripts/resource.h.in new file mode 100644 index 000000000..b27c3c882 --- /dev/null +++ b/cmake/scripts/resource.h.in @@ -0,0 +1,26 @@ +// this file is autogenerated, do not modify!!! +#pragma once + +#include +#include + +namespace dev +{ +namespace eth +{ + +class ${ETH_RESOURCE_NAME} +{ +public: + ${ETH_RESOURCE_NAME}(); + std::string loadResourceAsString(std::string _name): + +private: + std::unordered_map m_resources; + std::unordered_map m_sizes; + +}; + +} +} + diff --git a/cmake/scripts/resources.cmake b/cmake/scripts/resources.cmake index 2c92103ff..3e47a1bfb 100644 --- a/cmake/scripts/resources.cmake +++ b/cmake/scripts/resources.cmake @@ -1,37 +1,60 @@ - -# cmake -DETH_RES_FILE=... -DETH_DST_NAME=... -P scripts/resources.cmake -# cmake -DETH_RES_FILE=test.cmake -DETH_DST_NAME=dst -P resources.cmake +# based on: http://stackoverflow.com/questions/11813271/embed-resources-eg-shader-code-images-into-executable-library-with-cmake +# +# example: +# cmake -DETH_RES_FILE=test.cmake -P resources.cmake +# +# where test.cmake is: +# +# # BEGIN OF cmake.test +# +# set(copydlls "copydlls.cmake") +# set(conf "configure.cmake") +# +# # this three properties must be set! +# +# set(ETH_RESOURCE_NAME "EthResources") +# set(ETH_RESOURCE_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}") +# set(ETH_RESOURCES "copydlls" "conf") +# +# # END of cmake.test +# # should define ETH_RESOURCES include(${ETH_RES_FILE}) -set(ETH_RESULT_DATA) -set(ETH_RESULT_INIT) +set(ETH_RESULT_DATA "") +set(ETH_RESULT_INIT "") # resource is a name visible for cpp application foreach(resource ${ETH_RESOURCES}) - + # filename is the name of file which will be used in app set(filename ${${resource}}) # filedata is a file content file(READ ${filename} filedata HEX) + # read full name of the file + file(GLOB filename ${filename}) + # Convert hex data for C compatibility string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) # append static variables to result variable - list(APPEND ${ETH_RESULT_DATA} "static const unsigned char eth_${resource}[] = {\n // ${filename}\n ${filedata}\n};\n") + set(ETH_RESULT_DATA "${ETH_RESULT_DATA} static const unsigned char eth_${resource}[] = {\n // ${filename}\n ${filedata}\n};\n") # append init resources - list(APPEND ${ETH_RESULT_INIT} " eth_resources[\"${resource}\"] = (char const*)eth_${resource};\n") - list(APPEND ${ETH_RESULT_INIT} " eth_sizes[\"${resource}\"] = sizeof(eth_${resource});\n") + set(ETH_RESULT_INIT "${ETH_RESULT_INIT} eth_resources[\"${resource}\"] = (char const*)eth_${resource};\n") + set(ETH_RESULT_INIT "${ETH_RESULT_INIT} eth_sizes[\"${resource}\"] = sizeof(eth_${resource});\n") endforeach(resource) -configure_file("resource.cpp.in" "${ETH_DST_NAME}.cpp.tmp") +set(ETH_DST_NAME "${ETH_RESOURCE_LOCATION}/${ETH_RESOURCE_NAME}") + +configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.cpp.in" "${ETH_DST_NAME}.cpp.tmp") +configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.h.in" "${ETH_DST_NAME}.h.tmp") -include("../EthUtils.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/../EthUtils.cmake") replace_if_different("${ETH_DST_NAME}.cpp.tmp" "${ETH_DST_NAME}.cpp") -replace_if_different("resource.h" "${ETH_DST_NAME}.h") +replace_if_different("${ETH_DST_NAME}.h.tmp" "${ETH_DST_NAME}.h") diff --git a/cmake/scripts/test.cmake b/cmake/scripts/test.cmake deleted file mode 100644 index 883ed324e..000000000 --- a/cmake/scripts/test.cmake +++ /dev/null @@ -1,6 +0,0 @@ - -set(copydlls "copydlls.cmake") -set(conf "configure.cmake") - -set(ETH_RESOURCES "copyddls" "conf") - From 6da6f4ee53e884fe210ce8804c772c98971db324 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Wed, 29 Apr 2015 23:09:40 +0200 Subject: [PATCH 03/21] fixed indention --- cmake/scripts/resources.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/scripts/resources.cmake b/cmake/scripts/resources.cmake index 3e47a1bfb..d69d99e96 100644 --- a/cmake/scripts/resources.cmake +++ b/cmake/scripts/resources.cmake @@ -34,8 +34,8 @@ foreach(resource ${ETH_RESOURCES}) # filedata is a file content file(READ ${filename} filedata HEX) - # read full name of the file - file(GLOB filename ${filename}) + # read full name of the file + file(GLOB filename ${filename}) # Convert hex data for C compatibility string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) From 6315351b7af0377fd56c1022ff4991779838111e Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 30 Apr 2015 00:57:58 +0200 Subject: [PATCH 04/21] fixes for eth_add_resources --- cmake/EthUtils.cmake | 6 +++++- cmake/scripts/resource.cpp.in | 23 ---------------------- cmake/scripts/resource.h.in | 26 ------------------------- cmake/scripts/resource.hpp.in | 36 +++++++++++++++++++++++++++++++++++ cmake/scripts/resources.cmake | 10 ++++------ 5 files changed, 45 insertions(+), 56 deletions(-) delete mode 100644 cmake/scripts/resource.cpp.in delete mode 100644 cmake/scripts/resource.h.in create mode 100644 cmake/scripts/resource.hpp.in diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake index 147ce0737..d5da866ea 100644 --- a/cmake/EthUtils.cmake +++ b/cmake/EthUtils.cmake @@ -64,7 +64,11 @@ endmacro() # Creates C resources file from files function(eth_add_resources TARGET RESOURCE_FILE) - add_custom_command(TARGET ${TARGET} PRE_BUILD + + add_custom_target("${TARGET}.resources" COMMAND ${CMAKE_COMMAND} -DETH_RES_FILE="${RESOURCE_FILE}" -P "${ETH_SCRIPTS_DIR}/resources.cmake" ) + + add_dependencies(${TARGET} "${TARGET}.resources") + endfunction() diff --git a/cmake/scripts/resource.cpp.in b/cmake/scripts/resource.cpp.in deleted file mode 100644 index b73a8df1a..000000000 --- a/cmake/scripts/resource.cpp.in +++ /dev/null @@ -1,23 +0,0 @@ -// this file is autogenerated, do not modify!!! - -#include -#include -#include "${ETH_RESOURCE_NAME}.h" - -using namespace std; -using namespace dev; -using namespace dev::eth; - -${ETH_RESOURCE_NAME}::${ETH_RESOURCE_NAME}() -{ -${ETH_RESULT_DATA} -${ETH_RESULT_INIT} -} - -string ${ETH_RESOURCE_NAME}::loadResourceAsString(string _name) -{ - ostringstream bistream; - bistream.write(eth_resources[_name], eth_sizes[_name]); - return bistream.str(); -} - diff --git a/cmake/scripts/resource.h.in b/cmake/scripts/resource.h.in deleted file mode 100644 index b27c3c882..000000000 --- a/cmake/scripts/resource.h.in +++ /dev/null @@ -1,26 +0,0 @@ -// this file is autogenerated, do not modify!!! -#pragma once - -#include -#include - -namespace dev -{ -namespace eth -{ - -class ${ETH_RESOURCE_NAME} -{ -public: - ${ETH_RESOURCE_NAME}(); - std::string loadResourceAsString(std::string _name): - -private: - std::unordered_map m_resources; - std::unordered_map m_sizes; - -}; - -} -} - diff --git a/cmake/scripts/resource.hpp.in b/cmake/scripts/resource.hpp.in new file mode 100644 index 000000000..a8bbca377 --- /dev/null +++ b/cmake/scripts/resource.hpp.in @@ -0,0 +1,36 @@ +// this file is autogenerated, do not modify!!! +#pragma once + +#include +#include +#include + +namespace dev +{ +namespace eth +{ + +class ${ETH_RESOURCE_NAME} +{ +public: + ${ETH_RESOURCE_NAME}() + { +${ETH_RESULT_DATA} +${ETH_RESULT_INIT} + } + + std::string loadResourceAsString(std::string _name) + { + std::ostringstream bistream; + bistream.write(m_resources[_name], m_sizes[_name]); + return bistream.str(); + } + +private: + std::map m_resources; + std::map m_sizes; +}; + +} +} + diff --git a/cmake/scripts/resources.cmake b/cmake/scripts/resources.cmake index d69d99e96..93326a257 100644 --- a/cmake/scripts/resources.cmake +++ b/cmake/scripts/resources.cmake @@ -44,17 +44,15 @@ foreach(resource ${ETH_RESOURCES}) set(ETH_RESULT_DATA "${ETH_RESULT_DATA} static const unsigned char eth_${resource}[] = {\n // ${filename}\n ${filedata}\n};\n") # append init resources - set(ETH_RESULT_INIT "${ETH_RESULT_INIT} eth_resources[\"${resource}\"] = (char const*)eth_${resource};\n") - set(ETH_RESULT_INIT "${ETH_RESULT_INIT} eth_sizes[\"${resource}\"] = sizeof(eth_${resource});\n") + set(ETH_RESULT_INIT "${ETH_RESULT_INIT} m_resources[\"${resource}\"] = (char const*)eth_${resource};\n") + set(ETH_RESULT_INIT "${ETH_RESULT_INIT} m_sizes[\"${resource}\"] = sizeof(eth_${resource});\n") endforeach(resource) set(ETH_DST_NAME "${ETH_RESOURCE_LOCATION}/${ETH_RESOURCE_NAME}") -configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.cpp.in" "${ETH_DST_NAME}.cpp.tmp") -configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.h.in" "${ETH_DST_NAME}.h.tmp") +configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.hpp.in" "${ETH_DST_NAME}.hpp.tmp") include("${CMAKE_CURRENT_LIST_DIR}/../EthUtils.cmake") -replace_if_different("${ETH_DST_NAME}.cpp.tmp" "${ETH_DST_NAME}.cpp") -replace_if_different("${ETH_DST_NAME}.h.tmp" "${ETH_DST_NAME}.h") +replace_if_different("${ETH_DST_NAME}.hpp.tmp" "${ETH_DST_NAME}.hpp") From 0cbdda18d18f59fc05c3b2e84097fcf223da8de5 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 30 Apr 2015 10:43:30 +0200 Subject: [PATCH 05/21] simplified loading string from eth resources --- cmake/scripts/resource.hpp.in | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cmake/scripts/resource.hpp.in b/cmake/scripts/resource.hpp.in index a8bbca377..6a9740616 100644 --- a/cmake/scripts/resource.hpp.in +++ b/cmake/scripts/resource.hpp.in @@ -2,7 +2,6 @@ #pragma once #include -#include #include namespace dev @@ -19,12 +18,7 @@ ${ETH_RESULT_DATA} ${ETH_RESULT_INIT} } - std::string loadResourceAsString(std::string _name) - { - std::ostringstream bistream; - bistream.write(m_resources[_name], m_sizes[_name]); - return bistream.str(); - } + std::string loadResourceAsString(std::string _name) { return std::string(m_resources[_name], m_sizes[_name]); } private: std::map m_resources; From d4373dc185460790e280a35ff3ebe23a13816266 Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 30 Apr 2015 12:06:56 +0200 Subject: [PATCH 06/21] deployment fixed to work with GlobalRegistrar --- alethzero/DappLoader.cpp | 2 +- mix/qml/js/NetworkDeployment.js | 28 +++++++++++++++++++++++++--- mix/qml/js/TransactionHelper.js | 4 ++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/alethzero/DappLoader.cpp b/alethzero/DappLoader.cpp index 6ac1afbcb..7c754f8f5 100644 --- a/alethzero/DappLoader.cpp +++ b/alethzero/DappLoader.cpp @@ -82,7 +82,7 @@ DappLocation DappLoader::resolveAppUri(QString const& _uri) } string32 urlHintName = ZeroString32; - QByteArray utf8 = QString("UrlHint").toUtf8(); + QByteArray utf8 = QString("urlhint").toUtf8(); std::copy(utf8.data(), utf8.data() + utf8.size(), urlHintName.data()); Address urlHint = abiOut
(web3()->ethereum()->call(m_nameReg, abiIn("addr(bytes32)", urlHintName)).output); diff --git a/mix/qml/js/NetworkDeployment.js b/mix/qml/js/NetworkDeployment.js index fa85ddc54..9bbed2415 100644 --- a/mix/qml/js/NetworkDeployment.js +++ b/mix/qml/js/NetworkDeployment.js @@ -219,7 +219,9 @@ function finalizeDeployment(deploymentId, addresses) { function checkEthPath(dappUrl, callBack) { if (dappUrl.length === 1) - registerContentHash(deploymentDialog.eth, callBack); // we directly create a dapp under the root registrar. + reserve(deploymentDialog.eth, function() { + registerContentHash(deploymentDialog.eth, callBack); // we directly create a dapp under the root registrar. + }); else { // the first owned registrar must have been created to follow the path. @@ -310,7 +312,7 @@ function checkRegistration(dappUrl, addr, callBack) requests.push({ jsonrpc: "2.0", method: "eth_sendTransaction", - params: [ { "from": deploymentDialog.currentAccount, "gas": 20000, "code": "0x600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000163317815561058990819061003990396000f3007c010000000000000000000000000000000000000000000000000000000060003504630198489281146100a757806302571be3146100d957806321f8a721146100e35780632dff6941146100ed5780633b3b57de1461010d5780635a3a05bd1461013d5780635fd4b08a1461017057806389a69c0e1461017c578063b5c645bd146101b0578063be99a9801461022c578063c3d014d614610264578063d93e75731461029857005b73ffffffffffffffffffffffffffffffffffffffff600435166000908152600160205260409020548060005260206000f35b6000808052602081f35b6000808052602081f35b600435600090815260026020819052604090912001548060005260206000f35b600435600090815260026020908152604082205473ffffffffffffffffffffffffffffffffffffffff1680835291f35b600435600090815260026020908152604082206001015473ffffffffffffffffffffffffffffffffffffffff1680835291f35b60008060005260206000f35b6000546102c89060043590602435903373ffffffffffffffffffffffffffffffffffffffff908116911614610451576104b1565b600435600090815260026020819052604090912080546001820154919092015473ffffffffffffffffffffffffffffffffffffffff9283169291909116908273ffffffffffffffffffffffffffffffffffffffff166000528173ffffffffffffffffffffffffffffffffffffffff166020528060405260606000f35b6000546102ce906004359060243590604435903373ffffffffffffffffffffffffffffffffffffffff9081169116146104b557610584565b6000546102d49060043590602435903373ffffffffffffffffffffffffffffffffffffffff9081169116146102e05761031d565b6000546102da90600435903373ffffffffffffffffffffffffffffffffffffffff9081169116146103215761044e565b60006000f35b60006000f35b60006000f35b60006000f35b600082815260026020819052604080832090910183905583917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b5050565b60008181526002602090815260408083205473ffffffffffffffffffffffffffffffffffffffff1683526001909152902054811461035e576103de565b6000818152600260205260408082205473ffffffffffffffffffffffffffffffffffffffff169183917ff63780e752c6a54a94fc52715dbc5518a3b4c3c2833d301a204226548a2a85459190a360008181526002602090815260408083205473ffffffffffffffffffffffffffffffffffffffff16835260019091528120555b600081815260026020819052604080832080547fffffffffffffffffffffffff00000000000000000000000000000000000000009081168255600182018054909116905590910182905582917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b50565b60008281526002602052604080822060010180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168417905583917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b5050565b600083815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016831790558061051c57827fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc60006040a2610583565b73ffffffffffffffffffffffffffffffffffffffff8216837ff63780e752c6a54a94fc52715dbc5518a3b4c3c2833d301a204226548a2a854560006040a373ffffffffffffffffffffffffffffffffffffffff821660009081526001602052604090208390555b5b50505056" } ], + params: [ { "from": deploymentDialog.currentAccount, "gas": 20000, "code": "0x600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000163317815561058990819061003990396000f3007c010000000000000000000000000000000000000000000000000000000060003504630198489281146100a757806302571be3146100d957806321f8a721146100e35780632dff6941146100ed5780633b3b57de1461010d5780635a3a05bd1461013d5780635fd4b08a1461017057806389a69c0e1461017c578063b5c645bd146101b0578063be99a9801461022c578063c3d014d614610264578063d93e75731461029857005b73ffffffffffffffffffffffffffffffffffffffff600435166000908152600160205260409020548060005260206000f35b6000808052602081f35b6000808052602081f35b600435600090815260026020819052604090912001548060005260206000f35b600435600090815260026020908152604082205473ffffffffffffffffffffffffffffffffffffffff1680835291f35b600435600090815260026020908152604082206001015473ffffffffffffffffffffffffffffffffffffffff1680835291f35b60008060005260206000f35b6000546102c89060043590602435903373ffffffffffffffffffffffffffffffffffffffff90811691161461052557610585565b600435600090815260026020819052604090912080546001820154919092015473ffffffffffffffffffffffffffffffffffffffff9283169291909116908273ffffffffffffffffffffffffffffffffffffffff166000528173ffffffffffffffffffffffffffffffffffffffff166020528060405260606000f35b6000546102ce906004359060243590604435903373ffffffffffffffffffffffffffffffffffffffff9081169116146102e0576103af565b6000546102d49060043590602435903373ffffffffffffffffffffffffffffffffffffffff9081169116146103b4576103f1565b6000546102da90600435903373ffffffffffffffffffffffffffffffffffffffff9081169116146103f557610522565b60006000f35b60006000f35b60006000f35b60006000f35b600083815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016831790558061034757827fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc60006040a26103ae565b73ffffffffffffffffffffffffffffffffffffffff8216837ff63780e752c6a54a94fc52715dbc5518a3b4c3c2833d301a204226548a2a854560006040a373ffffffffffffffffffffffffffffffffffffffff821660009081526001602052604090208390555b5b505050565b600082815260026020819052604080832090910183905583917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b5050565b60008181526002602090815260408083205473ffffffffffffffffffffffffffffffffffffffff16835260019091529020548114610432576104b2565b6000818152600260205260408082205473ffffffffffffffffffffffffffffffffffffffff169183917ff63780e752c6a54a94fc52715dbc5518a3b4c3c2833d301a204226548a2a85459190a360008181526002602090815260408083205473ffffffffffffffffffffffffffffffffffffffff16835260019091528120555b600081815260026020819052604080832080547fffffffffffffffffffffffff00000000000000000000000000000000000000009081168255600182018054909116905590910182905582917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b50565b60008281526002602052604080822060010180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168417905583917fa6697e974e6a320f454390be03f74955e8978f1a6971ea6730542e37b66179bc91a25b505056" } ], id: jsonRpcRequestId++ }); @@ -353,6 +355,26 @@ function trCountIncrementTimeOut() deploymentError(error); } +function reserve(registrar, callBack) +{ + var txt = qsTr("Making reservation in the root registrar..."); + deploymentStepChanged(txt); + console.log(txt); + var requests = []; + var paramTitle = clientModel.encodeStringParam(projectModel.projectTitle); + requests.push({ + //reserve() + jsonrpc: "2.0", + method: "eth_sendTransaction", + params: [ { "from": deploymentDialog.currentAccount, "gas": "0xfffff", "to": '0x' + registrar, "data": "0x432ced04" + paramTitle } ], + id: jsonRpcRequestId++ + }); + rpcCall(requests, function (httpRequest, response) { + callBack(); + }); +} + + function registerContentHash(registrar, callBack) { var txt = qsTr("Finalizing Dapp registration ..."); @@ -398,7 +420,7 @@ function registerToUrlHint() function urlHintAddress(callBack) { var requests = []; - var urlHint = clientModel.encodeStringParam("UrlHint"); + var urlHint = clientModel.encodeStringParam("urlhint"); requests.push({ //registrar: get UrlHint addr jsonrpc: "2.0", diff --git a/mix/qml/js/TransactionHelper.js b/mix/qml/js/TransactionHelper.js index f0b4991fc..be057917c 100644 --- a/mix/qml/js/TransactionHelper.js +++ b/mix/qml/js/TransactionHelper.js @@ -17,6 +17,7 @@ function rpcCall(requests, callBack) { var jsonRpcUrl = "http://localhost:8080"; var rpcRequest = JSON.stringify(requests); + console.log(rpcRequest); var httpRequest = new XMLHttpRequest(); httpRequest.open("POST", jsonRpcUrl, true); httpRequest.setRequestHeader("Content-type", "application/json"); @@ -31,7 +32,10 @@ function rpcCall(requests, callBack) deploymentError(errorText); } else + { + console.log(httpRequest.responseText); callBack(httpRequest.status, httpRequest.responseText) + } } } httpRequest.send(rpcRequest); From 2a9e2f663e46cb521f12abd981e505cfae48ecd0 Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 30 Apr 2015 12:40:02 +0200 Subject: [PATCH 07/21] improved eth_add_resources --- cmake/EthUtils.cmake | 19 ++++++++++++++----- cmake/scripts/resources.cmake | 1 - 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake index d5da866ea..85d8c0dc4 100644 --- a/cmake/EthUtils.cmake +++ b/cmake/EthUtils.cmake @@ -63,12 +63,21 @@ macro(eth_add_test NAME) endmacro() # Creates C resources file from files -function(eth_add_resources TARGET RESOURCE_FILE) +function(eth_add_resources RESOURCE_FILE OUT_FILE) + include("${RESOURCE_FILE}") + set(OUTPUT "${ETH_RESOURCE_LOCATION}/${ETH_RESOURCE_NAME}.hpp") + set(${OUT_FILE} "${OUTPUT}" PARENT_SCOPE) - add_custom_target("${TARGET}.resources" - COMMAND ${CMAKE_COMMAND} -DETH_RES_FILE="${RESOURCE_FILE}" -P "${ETH_SCRIPTS_DIR}/resources.cmake" - ) + set(filenames "${RESOURCE_FILE}") + list(APPEND filenames "${ETH_SCRIPTS_DIR}/resources.cmake") + foreach(resource ${ETH_RESOURCES}) + list(APPEND filenames "${${resource}}") + endforeach(resource) - add_dependencies(${TARGET} "${TARGET}.resources") + message(STATUS "filenames; ${filenames}") + add_custom_command(OUTPUT ${OUTPUT} + COMMAND ${CMAKE_COMMAND} -DETH_RES_FILE="${RESOURCE_FILE}" -P "${ETH_SCRIPTS_DIR}/resources.cmake" + DEPENDS ${filenames} + ) endfunction() diff --git a/cmake/scripts/resources.cmake b/cmake/scripts/resources.cmake index 93326a257..b0cadbf6d 100644 --- a/cmake/scripts/resources.cmake +++ b/cmake/scripts/resources.cmake @@ -55,4 +55,3 @@ configure_file("${CMAKE_CURRENT_LIST_DIR}/resource.hpp.in" "${ETH_DST_NAME}.hpp. include("${CMAKE_CURRENT_LIST_DIR}/../EthUtils.cmake") replace_if_different("${ETH_DST_NAME}.hpp.tmp" "${ETH_DST_NAME}.hpp") - From 9dfabc79665e9420c297823daaa2bb5ee2055a5e Mon Sep 17 00:00:00 2001 From: Marek Kotewicz Date: Thu, 30 Apr 2015 12:47:14 +0200 Subject: [PATCH 08/21] removed redundant log in cmake utils --- cmake/EthUtils.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/EthUtils.cmake b/cmake/EthUtils.cmake index 85d8c0dc4..a426b1218 100644 --- a/cmake/EthUtils.cmake +++ b/cmake/EthUtils.cmake @@ -74,8 +74,6 @@ function(eth_add_resources RESOURCE_FILE OUT_FILE) list(APPEND filenames "${${resource}}") endforeach(resource) - message(STATUS "filenames; ${filenames}") - add_custom_command(OUTPUT ${OUTPUT} COMMAND ${CMAKE_COMMAND} -DETH_RES_FILE="${RESOURCE_FILE}" -P "${ETH_SCRIPTS_DIR}/resources.cmake" DEPENDS ${filenames} From c4fb35d4190d6d1e996cb91869ad8793f559152c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Apr 2015 14:06:44 +0100 Subject: [PATCH 09/21] Fix for #1766 --- libethereum/Client.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index cb51bcb87..bab8f9555 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -646,7 +646,7 @@ void Client::tick() m_bq.tick(m_bc); m_lastTick = chrono::system_clock::now(); if (m_report.ticks == 15) - cnote << activityReport(); + clog(ClientTrace) << activityReport(); } } From fd5ea37e5569b531ef6678b035e0a40b3c0e6c63 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Apr 2015 14:07:29 +0100 Subject: [PATCH 10/21] Transaction nonce now "sorted". Fixes #1615 --- libdevcore/Common.h | 2 + libethcore/EthashAux.cpp | 6 +-- libethereum/ClientBase.cpp | 7 +++- libethereum/Transaction.h | 11 ++--- libethereum/TransactionQueue.cpp | 49 ++++++++++++++++++++--- libethereum/TransactionQueue.h | 7 +++- libweb3jsonrpc/WebThreeStubServerBase.cpp | 4 +- 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/libdevcore/Common.h b/libdevcore/Common.h index 49491d4cc..b2d48da98 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -52,6 +52,8 @@ using byte = uint8_t; #define DEV_QUOTED_HELPER(s) #s #define DEV_QUOTED(s) DEV_QUOTED_HELPER(s) +#define DEV_IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} + namespace dev { diff --git a/libethcore/EthashAux.cpp b/libethcore/EthashAux.cpp index 19a96f550..9cb4d9fad 100644 --- a/libethcore/EthashAux.cpp +++ b/libethcore/EthashAux.cpp @@ -40,8 +40,6 @@ using namespace chrono; using namespace dev; using namespace eth; -#define ETH_IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} - EthashAux* dev::eth::EthashAux::s_this = nullptr; EthashAux::~EthashAux() @@ -171,8 +169,8 @@ EthashAux::FullType EthashAux::full(h256 const& _seedHash, bytesRef _dest, bool boost::filesystem::rename(oldMemoFile, memoFile); } - ETH_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile)); - ETH_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile + ".info")); + DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile)); + DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove(oldMemoFile + ".info")); ethash_params p = params(_seedHash); assert(!_dest || _dest.size() >= p.full_size); // must be big enough. diff --git a/libethereum/ClientBase.cpp b/libethereum/ClientBase.cpp index 7c56bce4e..eba8dbc67 100644 --- a/libethereum/ClientBase.cpp +++ b/libethereum/ClientBase.cpp @@ -48,8 +48,11 @@ State ClientBase::asOf(BlockNumber _h) const void ClientBase::submitTransaction(Secret _secret, u256 _value, Address _dest, bytes const& _data, u256 _gas, u256 _gasPrice) { prepareForTransaction(); - - u256 n = postMine().transactionsFrom(toAddress(_secret)); + + auto a = toAddress(_secret); + u256 n = postMine().transactionsFrom(a); + cdebug << "submitTx: " << a << "postMine=" << n << "; tq=" << m_tq.maxNonce(a); + n = max(n, m_tq.maxNonce(a)); Transaction t(_value, _gasPrice, _gas, _dest, _data, n, _secret); m_tq.import(t.rlp()); diff --git a/libethereum/Transaction.h b/libethereum/Transaction.h index 7276493c2..09102e0ba 100644 --- a/libethereum/Transaction.h +++ b/libethereum/Transaction.h @@ -221,19 +221,14 @@ using Transactions = std::vector; /// Simple human-readable stream-shift operator. inline std::ostream& operator<<(std::ostream& _out, Transaction const& _t) { - _out << "{"; + _out << _t.sha3().abridged() << "{"; if (_t.receiveAddress()) _out << _t.receiveAddress().abridged(); else _out << "[CREATE]"; - _out << "/" << _t.nonce() << "$" << _t.value() << "+" << _t.gas() << "@" << _t.gasPrice(); - try - { - _out << "<-" << _t.sender().abridged(); - } - catch (...) {} - _out << " #" << _t.data().size() << "}"; + _out << "/" << _t.data().size() << "$" << _t.value() << "+" << _t.gas() << "@" << _t.gasPrice(); + _out << "<-" << _t.safeSender().abridged() << " #" << _t.nonce() << "}"; return _out; } diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 40eec1ac5..57429d32c 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -53,7 +53,7 @@ ImportResult TransactionQueue::import(bytesConstRef _transactionRLP, ImportCallb UpgradeGuard ul(l); // If valid, append to blocks. - m_current[h] = t; + insertCurrent_WITH_LOCK(make_pair(h, t)); m_known.insert(h); if (_cb) m_callbacks[h] = _cb; @@ -74,13 +74,54 @@ ImportResult TransactionQueue::import(bytesConstRef _transactionRLP, ImportCallb return ImportResult::Success; } +u256 TransactionQueue::maxNonce(Address const& _a) const +{ + cdebug << "txQ::maxNonce" << _a; + ReadGuard l(m_lock); + u256 ret = 0; + auto r = m_senders.equal_range(_a); + for (auto it = r.first; it != r.second; ++it) + { + cdebug << it->first << "1+" << m_current.at(it->second).nonce(); + DEV_IGNORE_EXCEPTIONS(ret = max(ret, m_current.at(it->second).nonce() + 1)); + } + return ret; +} + +void TransactionQueue::insertCurrent_WITH_LOCK(std::pair const& _p) +{ + cdebug << "txQ::insertCurrent" << _p.first << _p.second.sender() << _p.second.nonce(); + m_senders.insert(make_pair(_p.second.sender(), _p.first)); + m_current.insert(_p); +} + +bool TransactionQueue::removeCurrent_WITH_LOCK(h256 const& _txHash) +{ + cdebug << "txQ::removeCurrent" << _txHash; + if (m_current.count(_txHash)) + { + auto r = m_senders.equal_range(m_current[_txHash].sender()); + for (auto it = r.first; it != r.second; ++it) + if (it->second == _txHash) + { + cdebug << "=> sender" << it->first; + m_senders.erase(it); + break; + } + cdebug << "=> nonce" << m_current[_txHash].nonce(); + m_current.erase(_txHash); + return true; + } + return false; +} + void TransactionQueue::setFuture(std::pair const& _t) { WriteGuard l(m_lock); if (m_current.count(_t.first)) { - m_current.erase(_t.first); m_unknown.insert(make_pair(_t.second.sender(), _t)); + m_current.erase(_t.first); } } @@ -104,9 +145,7 @@ void TransactionQueue::drop(h256 const& _txHash) m_dropped.insert(_txHash); m_known.erase(_txHash); - if (m_current.count(_txHash)) - m_current.erase(_txHash); - else + if (!removeCurrent_WITH_LOCK(_txHash)) { for (auto i = m_unknown.begin(); i != m_unknown.end(); ++i) if (i->second.first == _txHash) diff --git a/libethereum/TransactionQueue.h b/libethereum/TransactionQueue.h index 3858949cc..16bc34641 100644 --- a/libethereum/TransactionQueue.h +++ b/libethereum/TransactionQueue.h @@ -56,6 +56,7 @@ public: std::map transactions() const { ReadGuard l(m_lock); return m_current; } std::pair items() const { ReadGuard l(m_lock); return std::make_pair(m_current.size(), m_unknown.size()); } + u256 maxNonce(Address const& _a) const; void setFuture(std::pair const& _t); void noteGood(std::pair const& _t); @@ -64,12 +65,16 @@ public: template Handler onReady(T const& _t) { return m_onReady.add(_t); } private: - mutable SharedMutex m_lock; ///< General lock. + void insertCurrent_WITH_LOCK(std::pair const& _p); + bool removeCurrent_WITH_LOCK(h256 const& _txHash); + + mutable SharedMutex m_lock; ///< General lock. std::set m_known; ///< Hashes of transactions in both sets. std::map m_current; ///< Map of SHA3(tx) to tx. std::multimap> m_unknown; ///< For transactions that have a future nonce; we map their sender address to the tx stuff, and insert once the sender has a valid TX. std::map> m_callbacks; ///< Called once. std::set m_dropped; ///< Transactions that have previously been dropped. + std::multimap m_senders; ///< Mapping from the sender address to the transaction hash; useful for determining the nonce of a given sender. Signal m_onReady; ///< Called when a subsequent call to import transactions will return a non-empty container. Be nice and exit fast. }; diff --git a/libweb3jsonrpc/WebThreeStubServerBase.cpp b/libweb3jsonrpc/WebThreeStubServerBase.cpp index 208351fc6..2a1427b16 100644 --- a/libweb3jsonrpc/WebThreeStubServerBase.cpp +++ b/libweb3jsonrpc/WebThreeStubServerBase.cpp @@ -505,7 +505,7 @@ string WebThreeStubServerBase::eth_sendTransaction(Json::Value const& _json) if (!t.gasPrice) t.gasPrice = 10 * dev::eth::szabo; // TODO: should be determined by user somehow. if (!t.gas) - t.gas = min(client()->gasLimitRemaining(), client()->balanceAt(t.from) / t.gasPrice); + t.gas = min(client()->gasLimitRemaining() / 5, client()->balanceAt(t.from) / t.gasPrice); if (m_accounts->isRealAccount(t.from)) authenticate(t, false); @@ -534,7 +534,7 @@ string WebThreeStubServerBase::eth_signTransaction(Json::Value const& _json) if (!t.gasPrice) t.gasPrice = 10 * dev::eth::szabo; // TODO: should be determined by user somehow. if (!t.gas) - t.gas = min(client()->gasLimitRemaining(), client()->balanceAt(t.from) / t.gasPrice); + t.gas = min(client()->gasLimitRemaining() / 5, client()->balanceAt(t.from) / t.gasPrice); if (m_accounts->isRealAccount(t.from)) authenticate(t, false); From fb8d05b06e88f3b8df2efa5a5fa0f112339ab98a Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Thu, 30 Apr 2015 14:24:21 +0100 Subject: [PATCH 11/21] Always commit to state DB in batches. --- libdevcrypto/OverlayDB.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libdevcrypto/OverlayDB.cpp b/libdevcrypto/OverlayDB.cpp index 91f73ad49..e8bd609b0 100644 --- a/libdevcrypto/OverlayDB.cpp +++ b/libdevcrypto/OverlayDB.cpp @@ -20,6 +20,7 @@ */ #include +#include #include #include "OverlayDB.h" using namespace std; @@ -38,19 +39,21 @@ void OverlayDB::commit() { if (m_db) { + ldb::WriteBatch batch; // cnote << "Committing nodes to disk DB:"; for (auto const& i: m_over) { // cnote << i.first << "#" << m_refCount[i.first]; if (m_refCount[i.first]) - m_db->Put(m_writeOptions, ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); + batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); } for (auto const& i: m_auxActive) if (m_aux.count(i)) { - m_db->Put(m_writeOptions, i.ref(), bytesConstRef(&m_aux[i])); + batch.Put(i.ref(), bytesConstRef(&m_aux[i])); m_aux.erase(i); } + m_db->Write(m_writeOptions, &batch); m_auxActive.clear(); m_aux.clear(); m_over.clear(); From 976c990ddba0c718cb23e23364190b5a1eaef45e Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Fri, 1 May 2015 11:13:50 +0100 Subject: [PATCH 12/21] Minimise write-locking of DB. Fixes #1676 --- libethereum/BlockChain.cpp | 90 +++++++++++++++++++------------------- libethereum/BlockDetails.h | 1 + libethereum/Client.cpp | 89 ++++++++++++++++++++++++++----------- libethereum/Client.h | 6 ++- 4 files changed, 113 insertions(+), 73 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 2c5e5c01c..a4d070171 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -25,6 +25,7 @@ #include #endif #include +#include #include #include #include @@ -442,6 +443,11 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import t.restart(); #endif + ldb::WriteBatch blocksBatch; + ldb::WriteBatch extrasBatch; + h256 newLastBlockHash; + unsigned newLastBlockNumber = 0; + u256 td; #if ETH_CATCH try @@ -470,6 +476,7 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import #if ETH_PARANOIA checkConsistency(); #endif + // All ok - insert into DB { // ensure parent is cached for later addition. @@ -478,19 +485,9 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import // This is safe in practice since the caches don't get flushed nearly often enough to be // done here. details(bi.parentHash); - WriteGuard l(x_details); - m_details[bi.hash()] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}); m_details[bi.parentHash].children.push_back(bi.hash()); } - { - WriteGuard l(x_logBlooms); - m_logBlooms[bi.hash()] = blb; - } - { - WriteGuard l(x_receipts); - m_receipts[bi.hash()] = br; - } #if ETH_TIMED_IMPORTS collation = t.elapsed(); @@ -498,15 +495,12 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import #endif { - ReadGuard l1(x_blocks); ReadGuard l2(x_details); - ReadGuard l4(x_receipts); - ReadGuard l5(x_logBlooms); - m_extrasDB->Put(m_writeOptions, toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.hash()].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(m_logBlooms[bi.hash()].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(m_receipts[bi.hash()].rlp())); - m_blocksDB->Put(m_writeOptions, toSlice(bi.hash()), (ldb::Slice)ref(_block)); + extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp())); + extrasBatch.Put(toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp())); + blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block)); } #if ETH_TIMED_IMPORTS @@ -552,8 +546,11 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import h256 last = currentHash(); if (td > details(last).totalDifficulty) { + // don't include bi.hash() in treeRoute, since it's not yet in details DB... + // just tack it on afterwards. unsigned commonIndex; - tie(route, common, commonIndex) = treeRoute(last, bi.hash()); + tie(route, common, commonIndex) = treeRoute(last, bi.parentHash); + route.push_back(bi.hash()); // Most of the time these two will be equal - only when we're doing a chain revert will they not be if (common != last) @@ -564,20 +561,24 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import // Go through ret backwards until hash != last.parent and update m_transactionAddresses, m_blockHashes for (auto i = route.rbegin(); i != route.rend() && *i != common; ++i) { - auto b = block(*i); - BlockInfo bi(b); + BlockInfo tbi; + if (*i == bi.hash()) + tbi = bi; + else + tbi = BlockInfo(block(*i)); + // Collate logs into blooms. h256s alteredBlooms; { - LogBloom blockBloom = bi.logBloom; - blockBloom.shiftBloom<3>(sha3(bi.coinbaseAddress.ref())); + LogBloom blockBloom = tbi.logBloom; + blockBloom.shiftBloom<3>(sha3(tbi.coinbaseAddress.ref())); // Pre-memoize everything we need before locking x_blocksBlooms - for (unsigned level = 0, index = (unsigned)bi.number; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) + for (unsigned level = 0, index = (unsigned)tbi.number; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) blocksBlooms(chunkId(level, index / c_bloomIndexSize)); WriteGuard l(x_blocksBlooms); - for (unsigned level = 0, index = (unsigned)bi.number; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) + for (unsigned level = 0, index = (unsigned)tbi.number; level < c_bloomIndexLevels; level++, index /= c_bloomIndexSize) { unsigned i = index / c_bloomIndexSize; unsigned o = index % c_bloomIndexSize; @@ -588,38 +589,26 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import // Collate transaction hashes and remember who they were. h256s newTransactionAddresses; { - RLP blockRLP(b); + bytes blockBytes; + RLP blockRLP(*i == bi.hash() ? _block : (blockBytes = block(*i))); TransactionAddress ta; - ta.blockHash = bi.hash(); - WriteGuard l(x_transactionAddresses); + ta.blockHash = tbi.hash(); for (ta.index = 0; ta.index < blockRLP[1].itemCount(); ++ta.index) - { - newTransactionAddresses.push_back(sha3(blockRLP[1][ta.index].data())); - m_transactionAddresses[newTransactionAddresses.back()] = ta; - } - } - { - WriteGuard l(x_blockHashes); - m_blockHashes[h256(bi.number)].value = bi.hash(); + extrasBatch.Put(toSlice(sha3(blockRLP[1][ta.index].data()), ExtraTransactionAddress), (ldb::Slice)dev::ref(ta.rlp())); } // Update database with them. ReadGuard l1(x_blocksBlooms); - ReadGuard l3(x_blockHashes); - ReadGuard l6(x_transactionAddresses); for (auto const& h: alteredBlooms) - m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[h].rlp())); - m_extrasDB->Put(m_writeOptions, toSlice(h256(bi.number), ExtraBlockHash), (ldb::Slice)dev::ref(m_blockHashes[h256(bi.number)].rlp())); - for (auto const& h: newTransactionAddresses) - m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraTransactionAddress), (ldb::Slice)dev::ref(m_transactionAddresses[h].rlp())); + extrasBatch.Put(toSlice(h, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[h].rlp())); + extrasBatch.Put(toSlice(h256(tbi.number), ExtraBlockHash), (ldb::Slice)dev::ref(BlockHash(tbi.hash()).rlp())); } // FINALLY! change our best hash. { - WriteGuard l(x_lastBlockHash); - m_lastBlockHash = bi.hash(); - m_lastBlockNumber = (unsigned)bi.number; - m_extrasDB->Put(m_writeOptions, ldb::Slice("best"), ldb::Slice((char const*)&(bi.hash()), 32)); + newLastBlockHash = bi.hash(); + newLastBlockNumber = (unsigned)bi.number; + extrasBatch.Put(ldb::Slice("best"), ldb::Slice((char const*)&(bi.hash()), 32)); } clog(BlockChainNote) << " Imported and best" << td << " (#" << bi.number << "). Has" << (details(bi.parentHash).children.size() - 1) << "siblings. Route:" << route; @@ -637,6 +626,15 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import clog(BlockChainChat) << " Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << ")"; } + m_blocksDB->Write(m_writeOptions, &blocksBatch); + m_extrasDB->Write(m_writeOptions, &extrasBatch); + + ETH_WRITE_GUARDED(x_lastBlockHash) + { + m_lastBlockHash = newLastBlockHash; + m_lastBlockNumber = newLastBlockNumber; + } + #if ETH_TIMED_IMPORTS checkBest = t.elapsed(); cnote << "Import took:" << total.elapsed(); diff --git a/libethereum/BlockDetails.h b/libethereum/BlockDetails.h index 572ed1888..0baacb4da 100644 --- a/libethereum/BlockDetails.h +++ b/libethereum/BlockDetails.h @@ -92,6 +92,7 @@ struct BlockReceipts struct BlockHash { BlockHash() {} + BlockHash(h256 const& _h): value(_h) {} BlockHash(RLP const& _r) { value = _r.toHash(); } bytes rlp() const { return dev::rlp(value); } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index bab8f9555..63120cfa3 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -36,6 +36,12 @@ using namespace dev; using namespace dev::eth; using namespace p2p; +namespace dev +{ +struct TimerHelper { TimerHelper(char const* _id): id(_id) {} ~TimerHelper() { cdebug << "Timer" << id << t.elapsed() << "s"; } boost::timer t; char const* id; }; +#define DEV_TIMED(S) for (::std::pair<::dev::TimerHelper, bool> __eth_t(#S, true); __eth_t.second; __eth_t.second = false) +} + VersionChecker::VersionChecker(string const& _dbPath): m_path(_dbPath.size() ? _dbPath : Defaults::dbPath()) { @@ -246,9 +252,13 @@ void Client::startedWorking() ETH_WRITE_GUARDED(x_preMine) m_preMine.sync(m_bc); - ETH_WRITE_GUARDED(x_postMine) - ETH_READ_GUARDED(x_preMine) + ETH_READ_GUARDED(x_preMine) + { + ETH_WRITE_GUARDED(x_working) + m_working = m_preMine; + ETH_WRITE_GUARDED(x_postMine) m_postMine = m_preMine; + } } void Client::doneWorking() @@ -257,9 +267,13 @@ void Client::doneWorking() // TODO: currently it contains keys for *all* blocks. Make it remove old ones. ETH_WRITE_GUARDED(x_preMine) m_preMine.sync(m_bc); - ETH_WRITE_GUARDED(x_postMine) - ETH_READ_GUARDED(x_preMine) + ETH_READ_GUARDED(x_preMine) + { + ETH_WRITE_GUARDED(x_working) + m_working = m_preMine; + ETH_WRITE_GUARDED(x_postMine) m_postMine = m_preMine; + } } void Client::killChain() @@ -453,18 +467,20 @@ ProofOfWork::WorkPackage Client::getWork() bool Client::submitWork(ProofOfWork::Solution const& _solution) { bytes newBlock; - { - WriteGuard l(x_postMine); - if (!m_postMine.completeMine(_solution)) + DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) + if (!m_working.completeMine(_solution)) return false; - newBlock = m_postMine.blockData(); - // OPTIMISE: very inefficient to not utilise the existing OverlayDB in m_postMine that contains all trie changes. + + ETH_READ_GUARDED(x_working) + { + DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) + m_postMine = m_working; + newBlock = m_working.blockData(); } + + // OPTIMISE: very inefficient to not utilise the existing OverlayDB in m_postMine that contains all trie changes. m_bq.import(&newBlock, m_bc, true); -/* - ImportRoute ir = m_bc.attemptImport(newBlock, m_stateDB); - if (!ir.first.empty()) - onChainChanged(ir);*/ + return true; } @@ -489,12 +505,16 @@ void Client::syncTransactionQueue() h256Set changeds; TransactionReceipts newPendingReceipts; - ETH_WRITE_GUARDED(x_postMine) - tie(newPendingReceipts, m_syncTransactionQueue) = m_postMine.sync(m_bc, m_tq, *m_gp); + DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) + tie(newPendingReceipts, m_syncTransactionQueue) = m_working.sync(m_bc, m_tq, *m_gp); if (newPendingReceipts.empty()) return; + ETH_READ_GUARDED(x_working) + DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) + m_postMine = m_working; + ETH_READ_GUARDED(x_postMine) for (size_t i = 0; i < newPendingReceipts.size(); i++) appendFromNewPending(newPendingReceipts[i], changeds, m_postMine.pending()[i].sha3()); @@ -519,7 +539,7 @@ void Client::onChainChanged(ImportRoute const& _ir) clog(ClientNote) << "Dead block:" << h; for (auto const& t: m_bc.transactions(h)) { - clog(ClientNote) << "Resubmitting transaction " << Transaction(t, CheckTransaction::None); + clog(ClientNote) << "Resubmitting dead-block transaction " << Transaction(t, CheckTransaction::None); m_tq.import(t, TransactionQueue::ImportCallback(), IfDropped::Retry); } } @@ -545,18 +565,32 @@ void Client::onChainChanged(ImportRoute const& _ir) // RESTART MINING - // LOCKS REALLY NEEDED? bool preChanged = false; - ETH_WRITE_GUARDED(x_preMine) - preChanged = m_preMine.sync(m_bc); + State newPreMine; + ETH_READ_GUARDED(x_preMine) + newPreMine = m_preMine; + + // TODO: use m_postMine to avoid re-evaluating our own blocks. + preChanged = newPreMine.sync(m_bc); + if (preChanged || m_postMine.address() != m_preMine.address()) { if (isMining()) cnote << "New block on chain."; - ETH_WRITE_GUARDED(x_postMine) - ETH_READ_GUARDED(x_preMine) - m_postMine = m_preMine; + ETH_WRITE_GUARDED(x_preMine) + m_preMine = newPreMine; + DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) + m_working = newPreMine; + ETH_READ_GUARDED(x_postMine) + for (auto const& t: m_postMine.pending()) + { + clog(ClientNote) << "Resubmitting post-mine transaction " << t; + m_tq.import(t.rlp(), TransactionQueue::ImportCallback(), IfDropped::Retry); + } + ETH_READ_GUARDED(x_working) DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) + m_postMine = m_working; + changeds.insert(PendingChangedFilter); onPostStateChanged(); @@ -575,9 +609,12 @@ void Client::onPostStateChanged() cnote << "Post state changed: Restarting mining..."; if (isMining() || remoteActive()) { + DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) + m_working.commitToMine(m_bc); + ETH_READ_GUARDED(x_working) { - WriteGuard l(x_postMine); - m_postMine.commitToMine(m_bc); + DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) + m_postMine = m_working; m_miningInfo = m_postMine.info(); } m_farm.setWork(m_miningInfo); @@ -695,7 +732,9 @@ eth::State Client::state(h256 _block) const eth::State Client::state(unsigned _txi) const { - return m_postMine.fromPending(_txi); + ETH_READ_GUARDED(x_postMine) + return m_postMine.fromPending(_txi); + assert(false); } void Client::flushTransactions() diff --git a/libethereum/Client.h b/libethereum/Client.h index 662d05fd6..90273968c 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -279,10 +279,12 @@ private: std::shared_ptr m_gp; ///< The gas pricer. OverlayDB m_stateDB; ///< Acts as the central point for the state database, so multiple States can share it. - mutable SharedMutex x_preMine; ///< Lock on the OverlayDB and other attributes of m_preMine. + mutable SharedMutex x_preMine; ///< Lock on m_preMine. State m_preMine; ///< The present state of the client. - mutable SharedMutex x_postMine; ///< Lock on the OverlayDB and other attributes of m_postMine. + mutable SharedMutex x_postMine; ///< Lock on m_postMine. State m_postMine; ///< The state of the client which we're mining (i.e. it'll have all the rewards added). + mutable SharedMutex x_working; ///< Lock on m_working. + State m_working; ///< The state of the client which we're mining (i.e. it'll have all the rewards added), while we're actually working on it. BlockInfo m_miningInfo; ///< The header we're attempting to mine on (derived from m_postMine). bool remoteActive() const; ///< Is there an active and valid remote worker? bool m_remoteWorking = false; ///< Has the remote worker recently been reset? From 1069295ec6e13211897f772266771ea3c1419aa8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 2 May 2015 14:12:22 +0100 Subject: [PATCH 13/21] Fix for chain fork instance. --- alethzero/MainWin.cpp | 49 ++++++++++++++++++++------------------ libdevcrypto/TrieDB.h | 40 ++++++++++++++++++------------- libethereum/BlockChain.cpp | 33 ++++++++++++------------- libethereum/State.cpp | 4 ++-- 4 files changed, 67 insertions(+), 59 deletions(-) diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 4963d2742..ff505d5f2 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1097,30 +1097,33 @@ void Main::refreshBlockChain() blockItem->setSelected(true); int n = 0; - auto b = bc.block(h); - for (auto const& i: RLP(b)[1]) - { - Transaction t(i.data(), CheckTransaction::Everything); - QString s = t.receiveAddress() ? - QString(" %2 %5> %3: %1 [%4]") - .arg(formatBalance(t.value()).c_str()) - .arg(render(t.safeSender())) - .arg(render(t.receiveAddress())) - .arg((unsigned)t.nonce()) - .arg(ethereum()->codeAt(t.receiveAddress()).size() ? '*' : '-') : - QString(" %2 +> %3: %1 [%4]") - .arg(formatBalance(t.value()).c_str()) - .arg(render(t.safeSender())) - .arg(render(right160(sha3(rlpList(t.safeSender(), t.nonce()))))) - .arg((unsigned)t.nonce()); - QListWidgetItem* txItem = new QListWidgetItem(s, ui->blocks); - auto hba = QByteArray((char const*)h.data(), h.size); - txItem->setData(Qt::UserRole, hba); - txItem->setData(Qt::UserRole + 1, n); - if (oldSelected == hba) - txItem->setSelected(true); - n++; + try { + auto b = bc.block(h); + for (auto const& i: RLP(b)[1]) + { + Transaction t(i.data(), CheckTransaction::Everything); + QString s = t.receiveAddress() ? + QString(" %2 %5> %3: %1 [%4]") + .arg(formatBalance(t.value()).c_str()) + .arg(render(t.safeSender())) + .arg(render(t.receiveAddress())) + .arg((unsigned)t.nonce()) + .arg(ethereum()->codeAt(t.receiveAddress()).size() ? '*' : '-') : + QString(" %2 +> %3: %1 [%4]") + .arg(formatBalance(t.value()).c_str()) + .arg(render(t.safeSender())) + .arg(render(right160(sha3(rlpList(t.safeSender(), t.nonce()))))) + .arg((unsigned)t.nonce()); + QListWidgetItem* txItem = new QListWidgetItem(s, ui->blocks); + auto hba = QByteArray((char const*)h.data(), h.size); + txItem->setData(Qt::UserRole, hba); + txItem->setData(Qt::UserRole + 1, n); + if (oldSelected == hba) + txItem->setSelected(true); + n++; + } } + catch (...) {} }; if (filters.empty()) diff --git a/libdevcrypto/TrieDB.h b/libdevcrypto/TrieDB.h index a707c30f0..c609f4b43 100644 --- a/libdevcrypto/TrieDB.h +++ b/libdevcrypto/TrieDB.h @@ -47,6 +47,11 @@ struct InvalidTrie: virtual dev::Exception {}; extern const h256 c_shaNull; extern const h256 EmptyTrie; +enum class Verification { + Skip, + Normal +}; + /** * @brief Merkle Patricia Tree "Trie": a modifed base-16 Radix tree. * This version uses a database backend. @@ -69,23 +74,26 @@ public: using DB = _DB; GenericTrieDB(DB* _db = nullptr): m_db(_db) {} - GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } + GenericTrieDB(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { open(_db, _root, _v); } ~GenericTrieDB() {} void open(DB* _db) { m_db = _db; } - void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); } + void open(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { m_db = _db; setRoot(_root, _v); } void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); } - void setRoot(h256 _root) + void setRoot(h256 const& _root, Verification _v = Verification::Normal) { m_root = _root; - if (m_root == c_shaNull && !m_db->exists(m_root)) - init(); + if (_v == Verification::Normal) + { + if (m_root == c_shaNull && !m_db->exists(m_root)) + init(); - /*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/ - if (!node(m_root).size()) - BOOST_THROW_EXCEPTION(RootNotFound()); + /*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/ + if (!node(m_root).size()) + BOOST_THROW_EXCEPTION(RootNotFound()); + } } /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). @@ -93,7 +101,7 @@ public: /// True if the trie is initialised but empty (i.e. that the DB contains the root node which is empty). bool isEmpty() const { return m_root == c_shaNull && node(m_root).size(); } - h256 root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; } // patch the root in the case of the empty trie. TODO: handle this properly. + h256 const& root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; } // patch the root in the case of the empty trie. TODO: handle this properly. void debugPrint() {} @@ -301,7 +309,7 @@ public: using KeyType = _KeyType; SpecificTrieDB(DB* _db = nullptr): Generic(_db) {} - SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {} + SpecificTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Generic(_db, _root, _v) {} std::string operator[](KeyType _k) const { return at(_k); } @@ -349,7 +357,7 @@ public: using DB = _DB; HashedGenericTrieDB(DB* _db = nullptr): Super(_db) {} - HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {} + HashedGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Super(_db, _root, _v) {} using Super::open; using Super::init; @@ -402,20 +410,20 @@ class FatGenericTrieDB: public GenericTrieDB public: FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {} - FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } + FatGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal) { open(_db, _root, _v); } - void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); } + void open(DB* _db, h256 _root, Verification _v = Verification::Normal) { Super::open(_db); m_secure.open(_db); setRoot(_root, _v); } void init() { Super::init(); m_secure.init(); syncRoot(); } - void setRoot(h256 _root) + void setRoot(h256 _root, Verification _v = Verification::Normal) { if (!m_secure.isNull()) Super::db()->removeAux(m_secure.root()); - m_secure.setRoot(_root); + m_secure.setRoot(_root, _v); auto rb = Super::db()->lookupAux(m_secure.root()); auto r = h256(rb); - Super::setRoot(r); + Super::setRoot(r, _v); } h256 root() const { return m_secure.root(); } diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index a4d070171..c0394db95 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -445,8 +445,8 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import ldb::WriteBatch blocksBatch; ldb::WriteBatch extrasBatch; - h256 newLastBlockHash; - unsigned newLastBlockNumber = 0; + h256 newLastBlockHash = currentHash(); + unsigned newLastBlockNumber = number(); u256 td; #if ETH_CATCH @@ -478,30 +478,27 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import #endif // All ok - insert into DB - { - // ensure parent is cached for later addition. - // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard - // together with an "ensureCachedWithUpdatableLock(l)" method. - // This is safe in practice since the caches don't get flushed nearly often enough to be - // done here. - details(bi.parentHash); - WriteGuard l(x_details); + + // ensure parent is cached for later addition. + // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard + // together with an "ensureCachedWithUpdatableLock(l)" method. + // This is safe in practice since the caches don't get flushed nearly often enough to be + // done here. + details(bi.parentHash); + ETH_WRITE_GUARDED(x_details) m_details[bi.parentHash].children.push_back(bi.hash()); - } #if ETH_TIMED_IMPORTS collation = t.elapsed(); t.restart(); #endif - { - ReadGuard l2(x_details); - extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp())); + blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block)); + ETH_READ_GUARDED(x_details) extrasBatch.Put(toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp())); - extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp())); - blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block)); - } + extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp())); #if ETH_TIMED_IMPORTS writing = t.elapsed(); diff --git a/libethereum/State.cpp b/libethereum/State.cpp index d5a54985e..83a78e1e8 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -152,7 +152,7 @@ State::State(OverlayDB const& _db, BlockChain const& _bc, h256 _h): State::State(State const& _s): m_db(_s.m_db), - m_state(&m_db, _s.m_state.root()), + m_state(&m_db, _s.m_state.root(), Verification::Skip), m_transactions(_s.m_transactions), m_receipts(_s.m_receipts), m_transactionSet(_s.m_transactionSet), @@ -184,7 +184,7 @@ void State::paranoia(std::string const& _when, bool _enforceRefs) const State& State::operator=(State const& _s) { m_db = _s.m_db; - m_state.open(&m_db, _s.m_state.root()); + m_state.open(&m_db, _s.m_state.root(), Verification::Skip); m_transactions = _s.m_transactions; m_receipts = _s.m_receipts; m_transactionSet = _s.m_transactionSet; From 8efa22d61df435089331d514909cf370406bc37c Mon Sep 17 00:00:00 2001 From: CJentzsch Date: Sat, 2 May 2015 15:46:35 +0200 Subject: [PATCH 14/21] build fix --- libethereum/Client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 63120cfa3..0643ea31a 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -735,6 +735,7 @@ eth::State Client::state(unsigned _txi) const ETH_READ_GUARDED(x_postMine) return m_postMine.fromPending(_txi); assert(false); + return State(); } void Client::flushTransactions() From 83f57b9ed0c60767c470fce79213767c8e4c93b6 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sat, 2 May 2015 14:50:17 +0100 Subject: [PATCH 15/21] Avoid more maps, more lookups and SHA3. --- libdevcrypto/MemoryDB.cpp | 8 ++++---- libdevcrypto/MemoryDB.h | 19 ++++++++----------- libdevcrypto/OverlayDB.cpp | 14 ++++++++------ libethereum/BlockChain.cpp | 2 +- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/libdevcrypto/MemoryDB.cpp b/libdevcrypto/MemoryDB.cpp index a207bb7d6..deabc760e 100644 --- a/libdevcrypto/MemoryDB.cpp +++ b/libdevcrypto/MemoryDB.cpp @@ -41,7 +41,7 @@ std::map MemoryDB::get() const return ret; } -std::string MemoryDB::lookup(h256 _h) const +std::string MemoryDB::lookup(h256 const& _h) const { auto it = m_over.find(_h); if (it != m_over.end()) @@ -54,7 +54,7 @@ std::string MemoryDB::lookup(h256 _h) const return std::string(); } -bool MemoryDB::exists(h256 _h) const +bool MemoryDB::exists(h256 const& _h) const { auto it = m_over.find(_h); if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))) @@ -62,7 +62,7 @@ bool MemoryDB::exists(h256 _h) const return false; } -void MemoryDB::insert(h256 _h, bytesConstRef _v) +void MemoryDB::insert(h256 const& _h, bytesConstRef _v) { m_over[_h] = _v.toString(); m_refCount[_h]++; @@ -71,7 +71,7 @@ void MemoryDB::insert(h256 _h, bytesConstRef _v) #endif } -bool MemoryDB::kill(h256 _h) +bool MemoryDB::kill(h256 const& _h) { if (m_refCount.count(_h)) { diff --git a/libdevcrypto/MemoryDB.h b/libdevcrypto/MemoryDB.h index 58b1339a5..b9c32f09f 100644 --- a/libdevcrypto/MemoryDB.h +++ b/libdevcrypto/MemoryDB.h @@ -47,25 +47,22 @@ public: void clear() { m_over.clear(); } std::map get() const; - std::string lookup(h256 _h) const; - bool exists(h256 _h) const; - void insert(h256 _h, bytesConstRef _v); - bool kill(h256 _h); + std::string lookup(h256 const& _h) const; + bool exists(h256 const& _h) const; + void insert(h256 const& _h, bytesConstRef _v); + bool kill(h256 const& _h); void purge(); - bytes lookupAux(h256 _h) const { auto h = aux(_h); return m_aux.count(h) ? m_aux.at(h) : bytes(); } - void removeAux(h256 _h) { m_auxActive.erase(aux(_h)); } - void insertAux(h256 _h, bytesConstRef _v) { auto h = aux(_h); m_auxActive.insert(h); m_aux[h] = _v.toBytes(); } + bytes lookupAux(h256 const& _h) const { try { return m_aux.at(_h).first; } catch (...) { return bytes(); } } + void removeAux(h256 const& _h) { m_aux[_h].second = false; } + void insertAux(h256 const& _h, bytesConstRef _v) { m_aux[_h] = make_pair(_v.toBytes(), true); } std::set keys() const; protected: - static h256 aux(h256 _k) { return h256(sha3(_k).ref().cropped(0, 24), h256::AlignLeft); } - std::map m_over; std::map m_refCount; - std::set m_auxActive; - std::map m_aux; + std::map> m_aux; mutable bool m_enforceRefs = false; }; diff --git a/libdevcrypto/OverlayDB.cpp b/libdevcrypto/OverlayDB.cpp index e8bd609b0..58efe8dde 100644 --- a/libdevcrypto/OverlayDB.cpp +++ b/libdevcrypto/OverlayDB.cpp @@ -47,14 +47,14 @@ void OverlayDB::commit() if (m_refCount[i.first]) batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); } - for (auto const& i: m_auxActive) - if (m_aux.count(i)) + for (auto const& i: m_aux) + if (i.second.second) { - batch.Put(i.ref(), bytesConstRef(&m_aux[i])); - m_aux.erase(i); + bytes b = i.first.asBytes(); + b.push_back(255); // for aux + batch.Put(bytesConstRef(&b), bytesConstRef(&i.second.first)); } m_db->Write(m_writeOptions, &batch); - m_auxActive.clear(); m_aux.clear(); m_over.clear(); m_refCount.clear(); @@ -67,7 +67,9 @@ bytes OverlayDB::lookupAux(h256 _h) const if (!ret.empty()) return ret; std::string v; - m_db->Get(m_readOptions, aux(_h).ref(), &v); + bytes b = _h.asBytes(); + b.push_back(255); // for aux + m_db->Get(m_readOptions, bytesConstRef(&b), &v); if (v.empty()) cwarn << "Aux not found: " << _h; return asBytes(v); diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index c0394db95..e7728f1d2 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -47,7 +47,7 @@ using namespace dev::eth; namespace js = json_spirit; #define ETH_CATCH 1 -#define ETH_TIMED_IMPORTS 0 +#define ETH_TIMED_IMPORTS 1 #ifdef _WIN32 const char* BlockChainDebug::name() { return EthBlue "8" EthWhite " <>"; } From 1b1c2e95d1bc99e0b8dabe3ebf68a7a050d021a8 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 3 May 2015 02:05:43 +0100 Subject: [PATCH 16/21] Trie maps consolidated for speed. --- libdevcore/Common.cpp | 2 +- libdevcrypto/MemoryDB.cpp | 52 +++++++++++++++++++++----------------- libdevcrypto/MemoryDB.h | 7 +++-- libdevcrypto/OverlayDB.cpp | 14 +++++----- libdevcrypto/TrieDB.h | 8 +++--- libethcore/Common.cpp | 2 +- 6 files changed, 44 insertions(+), 41 deletions(-) diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp index f27637dec..b3d70c538 100644 --- a/libdevcore/Common.cpp +++ b/libdevcore/Common.cpp @@ -27,7 +27,7 @@ using namespace dev; namespace dev { -char const* Version = "0.9.13"; +char const* Version = "0.9.14"; } diff --git a/libdevcrypto/MemoryDB.cpp b/libdevcrypto/MemoryDB.cpp index deabc760e..907e6abe6 100644 --- a/libdevcrypto/MemoryDB.cpp +++ b/libdevcrypto/MemoryDB.cpp @@ -32,22 +32,20 @@ const char* DBWarn::name() { return "TDB"; } std::map MemoryDB::get() const { - if (!m_enforceRefs) - return m_over; std::map ret; - for (auto const& i: m_refCount) - if (i.second) - ret.insert(*m_over.find(i.first)); + for (auto const& i: m_main) + if (!m_enforceRefs || i.second.second > 0) + ret.insert(make_pair(i.first, i.second.first)); return ret; } std::string MemoryDB::lookup(h256 const& _h) const { - auto it = m_over.find(_h); - if (it != m_over.end()) + auto it = m_main.find(_h); + if (it != m_main.end()) { - if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first))) - return it->second; + if (!m_enforceRefs || it->second.second > 0) + return it->second.first; // else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first)) // cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h; } @@ -56,27 +54,33 @@ std::string MemoryDB::lookup(h256 const& _h) const bool MemoryDB::exists(h256 const& _h) const { - auto it = m_over.find(_h); - if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))) + auto it = m_main.find(_h); + if (it != m_main.end() && (!m_enforceRefs || it->second.second > 0)) return true; return false; } void MemoryDB::insert(h256 const& _h, bytesConstRef _v) { - m_over[_h] = _v.toString(); - m_refCount[_h]++; + auto it = m_main.find(_h); + if (it != m_main.end()) + { + it->second.first = _v.toString(); + it->second.second++; + } + else + m_main[_h] = make_pair(_v.toString(), 1); #if ETH_PARANOIA - dbdebug << "INST" << _h << "=>" << m_refCount[_h]; + dbdebug << "INST" << _h << "=>" << m_main[_h].second; #endif } bool MemoryDB::kill(h256 const& _h) { - if (m_refCount.count(_h)) + if (m_main.count(_h)) { - if (m_refCount[_h] > 0) - --m_refCount[_h]; + if (m_main[_h].second > 0) + m_main[_h].second--; #if ETH_PARANOIA else { @@ -85,7 +89,7 @@ bool MemoryDB::kill(h256 const& _h) dbdebug << "NOKILL-WAS" << _h; return false; } - dbdebug << "KILL" << _h << "=>" << m_refCount[_h]; + dbdebug << "KILL" << _h << "=>" << m_main[_h].second; return true; } else @@ -101,16 +105,18 @@ bool MemoryDB::kill(h256 const& _h) void MemoryDB::purge() { - for (auto const& i: m_refCount) - if (!i.second) - m_over.erase(i.first); + for (auto it = m_main.begin(); it != m_main.end(); ) + if (it->second.second) + ++it; + else + it = m_main.erase(it); } set MemoryDB::keys() const { set ret; - for (auto const& i: m_refCount) - if (i.second && h128(i.first.ref().cropped(0, 16))) + for (auto const& i: m_main) + if (i.second.second) ret.insert(i.first); return ret; } diff --git a/libdevcrypto/MemoryDB.h b/libdevcrypto/MemoryDB.h index b9c32f09f..71428ecdb 100644 --- a/libdevcrypto/MemoryDB.h +++ b/libdevcrypto/MemoryDB.h @@ -44,7 +44,7 @@ class MemoryDB public: MemoryDB() {} - void clear() { m_over.clear(); } + void clear() { m_main.clear(); } // WARNING !!!! didn't originally clear m_refCount!!! std::map get() const; std::string lookup(h256 const& _h) const; @@ -60,8 +60,7 @@ public: std::set keys() const; protected: - std::map m_over; - std::map m_refCount; + std::map> m_main; std::map> m_aux; mutable bool m_enforceRefs = false; @@ -80,7 +79,7 @@ private: inline std::ostream& operator<<(std::ostream& _out, MemoryDB const& _m) { - for (auto i: _m.get()) + for (auto const& i: _m.get()) { _out << i.first << ": "; _out << RLP(i.second); diff --git a/libdevcrypto/OverlayDB.cpp b/libdevcrypto/OverlayDB.cpp index 58efe8dde..87c8a0927 100644 --- a/libdevcrypto/OverlayDB.cpp +++ b/libdevcrypto/OverlayDB.cpp @@ -41,11 +41,11 @@ void OverlayDB::commit() { ldb::WriteBatch batch; // cnote << "Committing nodes to disk DB:"; - for (auto const& i: m_over) + for (auto const& i: m_main) { -// cnote << i.first << "#" << m_refCount[i.first]; - if (m_refCount[i.first]) - batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); +// cnote << i.first << "#" << m_main[i.first].second; + if (i.second.second) + batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.first.data(), i.second.first.size())); } for (auto const& i: m_aux) if (i.second.second) @@ -56,8 +56,7 @@ void OverlayDB::commit() } m_db->Write(m_writeOptions, &batch); m_aux.clear(); - m_over.clear(); - m_refCount.clear(); + m_main.clear(); } } @@ -77,8 +76,7 @@ bytes OverlayDB::lookupAux(h256 _h) const void OverlayDB::rollback() { - m_over.clear(); - m_refCount.clear(); + m_main.clear(); } std::string OverlayDB::lookup(h256 _h) const diff --git a/libdevcrypto/TrieDB.h b/libdevcrypto/TrieDB.h index c609f4b43..ef628d20b 100644 --- a/libdevcrypto/TrieDB.h +++ b/libdevcrypto/TrieDB.h @@ -861,8 +861,8 @@ template bytes GenericTrieDB::mergeAt(RLP const& _orig, NibbleSli template void GenericTrieDB::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v) { -#if ETH_PARANOIA - tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash() : std::string()); +#if ETH_PARANOIA || !ETH_TRUE + tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash().abridged() : std::string()); #endif RLP r = _orig; @@ -1016,8 +1016,8 @@ template bytes GenericTrieDB::deleteAt(RLP const& _orig, NibbleSl template bool GenericTrieDB::deleteAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k) { -#if ETH_PARANOIA - tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash() : std::string()); +#if ETH_PARANOIA || !ETH_TRUE + tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash().abridged() : std::string()); #endif bytes b = _orig.isEmpty() ? bytes() : deleteAt(_orig.isList() ? _orig : RLP(node(_orig.toHash())), _k); diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index 01c8cfa4c..2c7601dbc 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -36,7 +36,7 @@ namespace eth { const unsigned c_protocolVersion = 60; -const unsigned c_minorProtocolVersion = 1; +const unsigned c_minorProtocolVersion = 2; const unsigned c_databaseBaseVersion = 9; #if ETH_FATDB const unsigned c_databaseVersionModifier = 1; From be6dd3b62b74d7e4f14c23f835b3cebacdc91389 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 3 May 2015 15:38:42 +0100 Subject: [PATCH 17/21] Optimisations and fixes for the BlockQueue. --- libethereum/BlockChain.cpp | 2 +- libethereum/BlockQueue.cpp | 51 ++++++++++++++++++++++++-------- libethereum/BlockQueue.h | 18 +++++------ libethereum/Client.cpp | 5 ++-- libethereum/Transaction.h | 3 +- libethereum/TransactionQueue.cpp | 46 +++++++++++++++++++++++----- libethereum/TransactionQueue.h | 4 +++ 7 files changed, 95 insertions(+), 34 deletions(-) diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index e7728f1d2..69078b400 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -303,7 +303,7 @@ LastHashes BlockChain::lastHashes(unsigned _n) const tuple BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max) { - _bq.tick(*this); +// _bq.tick(*this); vector blocks; _bq.drain(blocks, _max); diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp index 44ddda637..0cccf8e6b 100644 --- a/libethereum/BlockQueue.cpp +++ b/libethereum/BlockQueue.cpp @@ -74,17 +74,19 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo } UpgradeGuard ul(l); + invariants_WITH_LOCK(); // Check it's not in the future (void)_isOurs; if (bi.timestamp > (u256)time(0)/* && !_isOurs*/) { - m_future.insert(make_pair((unsigned)bi.timestamp, _block.toBytes())); + m_future.insert(make_pair((unsigned)bi.timestamp, make_pair(h, _block.toBytes()))); char buf[24]; time_t bit = (unsigned)bi.timestamp; if (strftime(buf, 24, "%X", localtime(&bit)) == 0) buf[0] = '\0'; // empty if case strftime fails cblockq << "OK - queued for future [" << bi.timestamp << "vs" << time(0) << "] - will wait until" << buf; + invariants_WITH_LOCK(); return ImportResult::FutureTime; } else @@ -94,6 +96,7 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo { m_knownBad.insert(bi.hash()); // bad parent; this is bad too, note it as such + invariants_WITH_LOCK(); return ImportResult::BadChain; } else if (!m_readySet.count(bi.parentHash) && !m_drainingSet.count(bi.parentHash) && !_bc.isKnown(bi.parentHash)) @@ -103,16 +106,18 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo m_unknown.insert(make_pair(bi.parentHash, make_pair(h, _block.toBytes()))); m_unknownSet.insert(h); + invariants_WITH_LOCK(); return ImportResult::UnknownParent; } else { // If valid, append to blocks. cblockq << "OK - ready for chain insertion."; - m_ready.push_back(_block.toBytes()); + m_ready.push_back(make_pair(h, _block.toBytes())); m_readySet.insert(h); + invariants_WITH_LOCK(); - noteReadyWithoutWriteGuard(h); + noteReady_WITH_LOCK(h); m_onReady(); return ImportResult::Success; } @@ -122,27 +127,33 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo bool BlockQueue::doneDrain(h256s const& _bad) { WriteGuard l(m_lock); + invariants_WITH_LOCK(); m_drainingSet.clear(); if (_bad.size()) { - vector old; + vector> old; swap(m_ready, old); for (auto& b: old) { - BlockInfo bi(b); + BlockInfo bi(b.second); if (m_knownBad.count(bi.parentHash)) - m_knownBad.insert(bi.hash()); + { + m_knownBad.insert(b.first); + m_readySet.erase(b.first); + } else m_ready.push_back(std::move(b)); } } m_knownBad += _bad; + // GAA!!!! NEVER EMPTY?!?!?! TODO: remove items from readySet! + invariants_WITH_LOCK(); return !m_readySet.empty(); } void BlockQueue::tick(BlockChain const& _bc) { - vector todo; + vector> todo; { UpgradableGuard l(m_lock); if (m_future.empty()) @@ -158,16 +169,18 @@ void BlockQueue::tick(BlockChain const& _bc) { UpgradeGuard l2(l); + invariants_WITH_LOCK(); auto end = m_future.lower_bound(t); for (auto i = m_future.begin(); i != end; ++i) todo.push_back(move(i->second)); m_future.erase(m_future.begin(), end); + invariants_WITH_LOCK(); } } cblockq << "Importing" << todo.size() << "past-future blocks."; for (auto const& b: todo) - import(&b, _bc); + import(&b.second, _bc); } template T advanced(T _t, unsigned _n) @@ -194,25 +207,34 @@ QueueStatus BlockQueue::blockStatus(h256 const& _h) const void BlockQueue::drain(std::vector& o_out, unsigned _max) { WriteGuard l(m_lock); + invariants_WITH_LOCK(); if (m_drainingSet.empty()) { o_out.resize(min(_max, m_ready.size())); for (unsigned i = 0; i < o_out.size(); ++i) - swap(o_out[i], m_ready[i]); + swap(o_out[i], m_ready[i].second); m_ready.erase(m_ready.begin(), advanced(m_ready.begin(), o_out.size())); for (auto const& bs: o_out) { - auto h = sha3(bs); + // TODO: @optimise use map rather than vector & set. + auto h = BlockInfo::headerHash(bs); m_drainingSet.insert(h); m_readySet.erase(h); } // swap(o_out, m_ready); // swap(m_drainingSet, m_readySet); } + invariants_WITH_LOCK(); } -void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) +void BlockQueue::invariants_WITH_LOCK() const { + assert(m_readySet.size() == m_ready.size()); +} + +void BlockQueue::noteReady_WITH_LOCK(h256 const& _good) +{ + invariants_WITH_LOCK(); list goodQueue(1, _good); while (!goodQueue.empty()) { @@ -220,7 +242,7 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) goodQueue.pop_front(); for (auto it = r.first; it != r.second; ++it) { - m_ready.push_back(it->second.second); + m_ready.push_back(it->second); auto newReady = it->second.first; m_unknownSet.erase(newReady); m_readySet.insert(newReady); @@ -228,16 +250,19 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) } m_unknown.erase(r.first, r.second); } + invariants_WITH_LOCK(); } void BlockQueue::retryAllUnknown() { + invariants_WITH_LOCK(); for (auto it = m_unknown.begin(); it != m_unknown.end(); ++it) { - m_ready.push_back(it->second.second); + m_ready.push_back(it->second); auto newReady = it->second.first; m_unknownSet.erase(newReady); m_readySet.insert(newReady); } m_unknown.clear(); + invariants_WITH_LOCK(); } diff --git a/libethereum/BlockQueue.h b/libethereum/BlockQueue.h index adcc6ab39..1b1612fb7 100644 --- a/libethereum/BlockQueue.h +++ b/libethereum/BlockQueue.h @@ -78,7 +78,7 @@ public: bool doneDrain(h256s const& _knownBad = h256s()); /// Notify the queue that the chain has changed and a new block has attained 'ready' status (i.e. is in the chain). - void noteReady(h256 _b) { WriteGuard l(m_lock); noteReadyWithoutWriteGuard(_b); } + void noteReady(h256 const& _b) { WriteGuard l(m_lock); noteReady_WITH_LOCK(_b); } /// Force a retry of all the blocks with unknown parents. void retryAllUnknown(); @@ -87,7 +87,7 @@ public: std::pair items() const { ReadGuard l(m_lock); return std::make_pair(m_ready.size(), m_unknown.size()); } /// Clear everything. - void clear() { WriteGuard l(m_lock); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); } + void clear() { WriteGuard l(m_lock); invariants_WITH_LOCK(); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); invariants_WITH_LOCK(); } /// Return first block with an unknown parent. h256 firstUnknown() const { ReadGuard l(m_lock); return m_unknownSet.size() ? *m_unknownSet.begin() : h256(); } @@ -101,18 +101,18 @@ public: template Handler onReady(T const& _t) { return m_onReady.add(_t); } private: - void noteReadyWithoutWriteGuard(h256 _b); - void notePresentWithoutWriteGuard(bytesConstRef _block); + void noteReady_WITH_LOCK(h256 const& _b); + void invariants_WITH_LOCK() const; mutable boost::shared_mutex m_lock; ///< General lock. - std::set m_readySet; ///< All blocks ready for chain-import. std::set m_drainingSet; ///< All blocks being imported. - std::vector m_ready; ///< List of blocks, in correct order, ready for chain-import. + std::set m_readySet; ///< All blocks ready for chain-import. + std::vector> m_ready; ///< List of blocks, in correct order, ready for chain-import. std::set m_unknownSet; ///< Set of all blocks whose parents are not ready/in-chain. - std::multimap> m_unknown; ///< For transactions that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears. - std::multimap m_future; ///< Set of blocks that are not yet valid. + std::multimap> m_unknown; ///< For blocks that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears. std::set m_knownBad; ///< Set of blocks that we know will never be valid. - Signal m_onReady; ///< Called when a subsequent call to import transactions will return a non-empty container. Be nice and exit fast. + std::multimap> m_future;///< Set of blocks that are not yet valid. + Signal m_onReady; ///< Called when a subsequent call to import blocks will return a non-empty container. Be nice and exit fast. }; } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 0643ea31a..8d049d404 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -582,11 +582,12 @@ void Client::onChainChanged(ImportRoute const& _ir) m_preMine = newPreMine; DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) m_working = newPreMine; +// Transactions ts = m_postMine.pending(); ETH_READ_GUARDED(x_postMine) for (auto const& t: m_postMine.pending()) { clog(ClientNote) << "Resubmitting post-mine transaction " << t; - m_tq.import(t.rlp(), TransactionQueue::ImportCallback(), IfDropped::Retry); + m_tq.import(t, TransactionQueue::ImportCallback(), IfDropped::Retry); } ETH_READ_GUARDED(x_working) DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) m_postMine = m_working; @@ -662,7 +663,7 @@ void Client::doWork() bool t = true; if (m_syncBlockQueue.compare_exchange_strong(t, false)) - syncBlockQueue(); + syncBlockQueue(); // GAAA!!!!! CALLED TOO OFTEN!!! t = true; if (m_syncTransactionQueue.compare_exchange_strong(t, false) && !m_remoteWorking) diff --git a/libethereum/Transaction.h b/libethereum/Transaction.h index 09102e0ba..4271a5f27 100644 --- a/libethereum/Transaction.h +++ b/libethereum/Transaction.h @@ -149,7 +149,7 @@ public: bytes rlp(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return s.out(); } /// @returns the SHA3 hash of the RLP serialisation of this transaction. - h256 sha3(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return dev::sha3(s.out()); } + h256 sha3(IncludeSignature _sig = WithSignature) const { if (_sig == WithSignature && m_hashWith) return m_hashWith; RLPStream s; streamRLP(s, _sig); auto ret = dev::sha3(s.out()); if (_sig == WithSignature) m_hashWith = ret; return ret; } /// @returns the amount of ETH to be transferred by this (message-call) transaction, in Wei. Synonym for endowment(). u256 value() const { return m_value; } @@ -211,6 +211,7 @@ private: bytes m_data; ///< The data associated with the transaction, or the initialiser if it's a creation transaction. SignatureStruct m_vrs; ///< The signature of the transaction. Encodes the sender. + mutable h256 m_hashWith; ///< Cached hash of transaction with signature. mutable Address m_sender; ///< Cached sender, determined from signature. mutable bigint m_gasRequired = 0; ///< Memoised amount required for the transaction to run. }; diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 57429d32c..1bfdf535a 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -38,26 +38,56 @@ ImportResult TransactionQueue::import(bytesConstRef _transactionRLP, ImportCallb UpgradableGuard l(m_lock); // TODO: keep old transactions around and check in State for nonce validity - if (m_known.count(h)) + auto ir = check_WITH_LOCK(h, _ik); + if (ir != ImportResult::Success) + return ir; + + Transaction t(_transactionRLP, CheckTransaction::Everything); + UpgradeGuard ul(l); + return manageImport_WITH_LOCK(h, t, _cb); +} + +ImportResult TransactionQueue::check_WITH_LOCK(h256 const& _h, IfDropped _ik) +{ + if (m_known.count(_h)) return ImportResult::AlreadyKnown; - if (m_dropped.count(h) && _ik == IfDropped::Ignore) + if (m_dropped.count(_h) && _ik == IfDropped::Ignore) return ImportResult::AlreadyInChain; + return ImportResult::Success; +} + +ImportResult TransactionQueue::import(Transaction const& _transaction, ImportCallback const& _cb, IfDropped _ik) +{ + // Check if we already know this transaction. + h256 h = _transaction.sha3(WithSignature); + + UpgradableGuard l(m_lock); + // TODO: keep old transactions around and check in State for nonce validity + + auto ir = check_WITH_LOCK(h, _ik); + if (ir != ImportResult::Success) + return ir; + + UpgradeGuard ul(l); + return manageImport_WITH_LOCK(h, _transaction, _cb); +} + +ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb) +{ try { // Check validity of _transactionRLP as a transaction. To do this we just deserialise and attempt to determine the sender. // If it doesn't work, the signature is bad. // The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction). - Transaction t(_transactionRLP, CheckTransaction::Everything); - UpgradeGuard ul(l); // If valid, append to blocks. - insertCurrent_WITH_LOCK(make_pair(h, t)); - m_known.insert(h); + insertCurrent_WITH_LOCK(make_pair(_h, _transaction)); + m_known.insert(_h); if (_cb) - m_callbacks[h] = _cb; - ctxq << "Queued vaguely legit-looking transaction" << h; + m_callbacks[_h] = _cb; + ctxq << "Queued vaguely legit-looking transaction" << _h; m_onReady(); } catch (Exception const& _e) diff --git a/libethereum/TransactionQueue.h b/libethereum/TransactionQueue.h index 16bc34641..69e1c935f 100644 --- a/libethereum/TransactionQueue.h +++ b/libethereum/TransactionQueue.h @@ -49,6 +49,7 @@ class TransactionQueue public: using ImportCallback = std::function; + ImportResult import(Transaction const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore); ImportResult import(bytes const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore) { return import(&_tx, _cb, _ik); } ImportResult import(bytesConstRef _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore); @@ -65,6 +66,9 @@ public: template Handler onReady(T const& _t) { return m_onReady.add(_t); } private: + ImportResult check_WITH_LOCK(h256 const& _h, IfDropped _ik); + ImportResult manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb); + void insertCurrent_WITH_LOCK(std::pair const& _p); bool removeCurrent_WITH_LOCK(h256 const& _txHash); From 1aa1950d75ba6414d3cf43c28f6368535a32244c Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 3 May 2015 20:45:23 +0100 Subject: [PATCH 18/21] Invariant utilities. --- libdevcore/Common.cpp | 8 +++++++- libdevcore/Common.h | 37 +++++++++++++++++++++++++++++++++++-- libdevcore/Exceptions.h | 1 + libethereum/BlockQueue.cpp | 26 ++++++++------------------ libethereum/BlockQueue.h | 8 +++++--- libethereum/Client.cpp | 3 +-- 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp index b3d70c538..3defa57b2 100644 --- a/libdevcore/Common.cpp +++ b/libdevcore/Common.cpp @@ -20,7 +20,7 @@ */ #include "Common.h" - +#include "Exceptions.h" using namespace std; using namespace dev; @@ -29,5 +29,11 @@ namespace dev char const* Version = "0.9.14"; +void HasInvariants::checkInvariants() const +{ + if (!invariants()) + BOOST_THROW_EXCEPTION(FailedInvariant()); +} + } diff --git a/libdevcore/Common.h b/libdevcore/Common.h index b2d48da98..03da4c8aa 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -128,14 +128,46 @@ inline N diff(N const& _a, N const& _b) } /// RAII utility class whose destructor calls a given function. -class ScopeGuard { +class ScopeGuard +{ public: ScopeGuard(std::function _f): m_f(_f) {} ~ScopeGuard() { m_f(); } + private: std::function m_f; }; +/// Inheritable for classes that have invariants. +class HasInvariants +{ +public: + /// Check invariants are met, throw if not. + void checkInvariants() const; + +protected: + /// Reimplement to specify the invariants. + virtual bool invariants() const = 0; +}; + +/// RAII checker for invariant assertions. +class InvariantChecker +{ +public: + InvariantChecker(HasInvariants* _this): m_this(_this) { m_this->checkInvariants(); } + ~InvariantChecker() { m_this->checkInvariants(); } + +private: + HasInvariants const* m_this; +}; + +/// Scope guard for invariant check in a class derived from HasInvariants. +#if ETH_DEBUG +#define DEV_INVARIANT_CHECK ::dev::InvariantChecker __dev_invariantCheck(this) +#else +#define DEV_INVARIANT_CHECK (void)0; +#endif + enum class WithExisting: int { Trust = 0, @@ -145,7 +177,8 @@ enum class WithExisting: int } -namespace std { +namespace std +{ inline dev::WithExisting max(dev::WithExisting _a, dev::WithExisting _b) { diff --git a/libdevcore/Exceptions.h b/libdevcore/Exceptions.h index 219047827..36c624a71 100644 --- a/libdevcore/Exceptions.h +++ b/libdevcore/Exceptions.h @@ -53,6 +53,7 @@ struct BadRoot: virtual Exception {}; struct FileError: virtual Exception {}; struct Overflow: virtual Exception {}; struct InterfaceNotSupported: virtual Exception { public: InterfaceNotSupported(std::string _f): Exception("Interface " + _f + " not supported.") {} }; +struct FailedInvariant: virtual Exception {}; // error information to be added to exceptions using errinfo_invalidSymbol = boost::error_info; diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp index 0cccf8e6b..43d2b4cb8 100644 --- a/libethereum/BlockQueue.cpp +++ b/libethereum/BlockQueue.cpp @@ -74,7 +74,7 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo } UpgradeGuard ul(l); - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; // Check it's not in the future (void)_isOurs; @@ -86,7 +86,6 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo if (strftime(buf, 24, "%X", localtime(&bit)) == 0) buf[0] = '\0'; // empty if case strftime fails cblockq << "OK - queued for future [" << bi.timestamp << "vs" << time(0) << "] - will wait until" << buf; - invariants_WITH_LOCK(); return ImportResult::FutureTime; } else @@ -96,7 +95,6 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo { m_knownBad.insert(bi.hash()); // bad parent; this is bad too, note it as such - invariants_WITH_LOCK(); return ImportResult::BadChain; } else if (!m_readySet.count(bi.parentHash) && !m_drainingSet.count(bi.parentHash) && !_bc.isKnown(bi.parentHash)) @@ -106,7 +104,6 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo m_unknown.insert(make_pair(bi.parentHash, make_pair(h, _block.toBytes()))); m_unknownSet.insert(h); - invariants_WITH_LOCK(); return ImportResult::UnknownParent; } else @@ -115,7 +112,6 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo cblockq << "OK - ready for chain insertion."; m_ready.push_back(make_pair(h, _block.toBytes())); m_readySet.insert(h); - invariants_WITH_LOCK(); noteReady_WITH_LOCK(h); m_onReady(); @@ -127,7 +123,7 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo bool BlockQueue::doneDrain(h256s const& _bad) { WriteGuard l(m_lock); - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; m_drainingSet.clear(); if (_bad.size()) { @@ -146,8 +142,6 @@ bool BlockQueue::doneDrain(h256s const& _bad) } } m_knownBad += _bad; - // GAA!!!! NEVER EMPTY?!?!?! TODO: remove items from readySet! - invariants_WITH_LOCK(); return !m_readySet.empty(); } @@ -169,12 +163,11 @@ void BlockQueue::tick(BlockChain const& _bc) { UpgradeGuard l2(l); - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; auto end = m_future.lower_bound(t); for (auto i = m_future.begin(); i != end; ++i) todo.push_back(move(i->second)); m_future.erase(m_future.begin(), end); - invariants_WITH_LOCK(); } } cblockq << "Importing" << todo.size() << "past-future blocks."; @@ -207,7 +200,7 @@ QueueStatus BlockQueue::blockStatus(h256 const& _h) const void BlockQueue::drain(std::vector& o_out, unsigned _max) { WriteGuard l(m_lock); - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; if (m_drainingSet.empty()) { o_out.resize(min(_max, m_ready.size())); @@ -224,17 +217,16 @@ void BlockQueue::drain(std::vector& o_out, unsigned _max) // swap(o_out, m_ready); // swap(m_drainingSet, m_readySet); } - invariants_WITH_LOCK(); } -void BlockQueue::invariants_WITH_LOCK() const +bool BlockQueue::invariants() const { - assert(m_readySet.size() == m_ready.size()); + return m_readySet.size() == m_ready.size(); } void BlockQueue::noteReady_WITH_LOCK(h256 const& _good) { - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; list goodQueue(1, _good); while (!goodQueue.empty()) { @@ -250,12 +242,11 @@ void BlockQueue::noteReady_WITH_LOCK(h256 const& _good) } m_unknown.erase(r.first, r.second); } - invariants_WITH_LOCK(); } void BlockQueue::retryAllUnknown() { - invariants_WITH_LOCK(); + DEV_INVARIANT_CHECK; for (auto it = m_unknown.begin(); it != m_unknown.end(); ++it) { m_ready.push_back(it->second); @@ -264,5 +255,4 @@ void BlockQueue::retryAllUnknown() m_readySet.insert(newReady); } m_unknown.clear(); - invariants_WITH_LOCK(); } diff --git a/libethereum/BlockQueue.h b/libethereum/BlockQueue.h index 1b1612fb7..d9cb0ed53 100644 --- a/libethereum/BlockQueue.h +++ b/libethereum/BlockQueue.h @@ -30,6 +30,7 @@ namespace dev { + namespace eth { @@ -60,7 +61,7 @@ enum class QueueStatus * Sorts them ready for blockchain insertion (with the BlockChain::sync() method). * @threadsafe */ -class BlockQueue +class BlockQueue: HasInvariants { public: /// Import a block into the queue. @@ -87,7 +88,7 @@ public: std::pair items() const { ReadGuard l(m_lock); return std::make_pair(m_ready.size(), m_unknown.size()); } /// Clear everything. - void clear() { WriteGuard l(m_lock); invariants_WITH_LOCK(); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); invariants_WITH_LOCK(); } + void clear() { WriteGuard l(m_lock); DEV_INVARIANT_CHECK; m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); } /// Return first block with an unknown parent. h256 firstUnknown() const { ReadGuard l(m_lock); return m_unknownSet.size() ? *m_unknownSet.begin() : h256(); } @@ -102,7 +103,8 @@ public: private: void noteReady_WITH_LOCK(h256 const& _b); - void invariants_WITH_LOCK() const; + + bool invariants() const override; mutable boost::shared_mutex m_lock; ///< General lock. std::set m_drainingSet; ///< All blocks being imported. diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 8d049d404..befebd765 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -582,7 +582,6 @@ void Client::onChainChanged(ImportRoute const& _ir) m_preMine = newPreMine; DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) m_working = newPreMine; -// Transactions ts = m_postMine.pending(); ETH_READ_GUARDED(x_postMine) for (auto const& t: m_postMine.pending()) { @@ -663,7 +662,7 @@ void Client::doWork() bool t = true; if (m_syncBlockQueue.compare_exchange_strong(t, false)) - syncBlockQueue(); // GAAA!!!!! CALLED TOO OFTEN!!! + syncBlockQueue(); t = true; if (m_syncTransactionQueue.compare_exchange_strong(t, false) && !m_remoteWorking) From 84ecbaff9e2540d06a5c1fccf547a528158dd0b1 Mon Sep 17 00:00:00 2001 From: Gav Wood Date: Sun, 3 May 2015 22:50:21 +0100 Subject: [PATCH 19/21] Condition variables rather than polling FTW. --- libethereum/Client.cpp | 7 ++++--- libethereum/Client.h | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index befebd765..e8ab0ff2f 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -658,8 +658,6 @@ void Client::noteChanged(h256Set const& _filters) void Client::doWork() { - // TODO: Use condition variable rather than this rubbish. - bool t = true; if (m_syncBlockQueue.compare_exchange_strong(t, false)) syncBlockQueue(); @@ -671,7 +669,10 @@ void Client::doWork() tick(); if (!m_syncBlockQueue && !m_syncTransactionQueue) - this_thread::sleep_for(chrono::milliseconds(20)); + { + std::unique_lock l(x_signalled); + m_signalled.wait_for(l, chrono::seconds(1)); + } } void Client::tick() diff --git a/libethereum/Client.h b/libethereum/Client.h index 90273968c..946825828 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -22,6 +22,7 @@ #pragma once #include +#include #include #include #include @@ -258,10 +259,10 @@ private: void syncTransactionQueue(); /// Magically called when m_tq needs syncing. Be nice and don't block. - void onTransactionQueueReady() { m_syncTransactionQueue = true; } + void onTransactionQueueReady() { m_syncTransactionQueue = true; m_signalled.notify_all(); } /// Magically called when m_tq needs syncing. Be nice and don't block. - void onBlockQueueReady() { m_syncBlockQueue = true; } + void onBlockQueueReady() { m_syncBlockQueue = true; m_signalled.notify_all(); } /// Called when the post state has changed (i.e. when more transactions are in it or we're mining on a new block). /// This updates m_miningInfo. @@ -309,6 +310,8 @@ private: ActivityReport m_report; // TODO!!!!!! REPLACE WITH A PROPER X-THREAD ASIO SIGNAL SYSTEM (could just be condition variables) + std::condition_variable m_signalled; + Mutex x_signalled; std::atomic m_syncTransactionQueue = {false}; std::atomic m_syncBlockQueue = {false}; }; From f0e9f6e0801e6351312200af91242ec592dcf392 Mon Sep 17 00:00:00 2001 From: subtly Date: Mon, 4 May 2015 02:25:23 +0100 Subject: [PATCH 20/21] Allow 9x ingress connections so non-nat clients can connect and use the network. To be reduced after greedy go is updated to discontinue filling up the network's connection capacity by accepting connections beyond the ones it didn't make and not connecting to every node on the entire network. --- libp2p/Host.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/Host.cpp b/libp2p/Host.cpp index 09802e445..030c406c0 100644 --- a/libp2p/Host.cpp +++ b/libp2p/Host.cpp @@ -226,7 +226,7 @@ void Host::startPeerSession(Public const& _id, RLP const& _rlp, RLPXFrameIO* _io return; } - if (peerCount() > 2 * m_idealPeerCount) + if (peerCount() > 9 * m_idealPeerCount) { ps->disconnect(TooManyPeers); return; @@ -341,7 +341,7 @@ void Host::runAcceptor() auto socket = make_shared(new bi::tcp::socket(m_ioService)); m_tcp4Acceptor.async_accept(socket->ref(), [=](boost::system::error_code ec) { - if (peerCount() > 2 * m_idealPeerCount) + if (peerCount() > 9 * m_idealPeerCount) { clog(NetConnect) << "Dropping incoming connect due to maximum peer count (2 * ideal peer count): " << socket->remoteEndpoint(); socket->close(); From 1e6ff441fe88ee5fc3ef1a9110184f1e983a76ba Mon Sep 17 00:00:00 2001 From: chriseth Date: Mon, 4 May 2015 10:16:10 +0200 Subject: [PATCH 21/21] Fix for broken continue in for loop. Fixes #1789. --- libsolidity/Compiler.cpp | 5 ++- test/libsolidity/SolidityEndToEndTest.cpp | 46 +++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/libsolidity/Compiler.cpp b/libsolidity/Compiler.cpp index 37b577ccd..d57f30c12 100644 --- a/libsolidity/Compiler.cpp +++ b/libsolidity/Compiler.cpp @@ -431,7 +431,8 @@ bool Compiler::visit(ForStatement const& _forStatement) CompilerContext::LocationSetter locationSetter(m_context, _forStatement); eth::AssemblyItem loopStart = m_context.newTag(); eth::AssemblyItem loopEnd = m_context.newTag(); - m_continueTags.push_back(loopStart); + eth::AssemblyItem loopNext = m_context.newTag(); + m_continueTags.push_back(loopNext); m_breakTags.push_back(loopEnd); if (_forStatement.getInitializationExpression()) @@ -449,6 +450,8 @@ bool Compiler::visit(ForStatement const& _forStatement) _forStatement.getBody().accept(*this); + m_context << loopNext; + // for's loop expression if existing if (_forStatement.getLoopExpression()) _forStatement.getLoopExpression()->accept(*this); diff --git a/test/libsolidity/SolidityEndToEndTest.cpp b/test/libsolidity/SolidityEndToEndTest.cpp index 24e5f7b4e..f168ad454 100644 --- a/test/libsolidity/SolidityEndToEndTest.cpp +++ b/test/libsolidity/SolidityEndToEndTest.cpp @@ -300,6 +300,52 @@ BOOST_AUTO_TEST_CASE(for_loop_simple_init_expr) testSolidityAgainstCppOnRange("f(uint256)", for_loop_simple_init_expr_cpp, 0, 5); } +BOOST_AUTO_TEST_CASE(for_loop_break_continue) +{ + char const* sourceCode = R"( + contract test { + function f(uint n) returns (uint r) + { + uint i = 1; + uint k = 0; + for (i *= 5; k < n; i *= 7) + { + k++; + i += 4; + if (n % 3 == 0) + break; + i += 9; + if (n % 2 == 0) + continue; + i += 19; + } + return i; + } + } + )"; + compileAndRun(sourceCode); + + auto breakContinue = [](u256 const& n) -> u256 + { + u256 i = 1; + u256 k = 0; + for (i *= 5; k < n; i *= 7) + { + k++; + i += 4; + if (n % 3 == 0) + break; + i += 9; + if (n % 2 == 0) + continue; + i += 19; + } + return i; + }; + + testSolidityAgainstCppOnRange("f(uint256)", breakContinue, 0, 10); +} + BOOST_AUTO_TEST_CASE(calling_other_functions) { char const* sourceCode = "contract collatz {\n"