diff --git a/libdevcore/Common.h b/libdevcore/Common.h index c7fdb6dab..edd14f4a5 100644 --- a/libdevcore/Common.h +++ b/libdevcore/Common.h @@ -64,6 +64,8 @@ using byte = uint8_t; #define DEV_IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} +#define DEV_IF_NO_ELSE(X) if(!(X)){}else + namespace dev { diff --git a/libdevcore/CommonData.h b/libdevcore/CommonData.h index ed09e60ee..0faba8e8e 100644 --- a/libdevcore/CommonData.h +++ b/libdevcore/CommonData.h @@ -50,8 +50,8 @@ enum class HexPrefix /// Convert a series of bytes to the corresponding string of hex duplets. /// @param _w specifies the width of the first of the elements. Defaults to two - enough to represent a byte. /// @example toHex("A\x69") == "4169" -template -std::string toHex(_T const& _data, int _w = 2, HexPrefix _prefix = HexPrefix::DontAdd) +template +std::string toHex(T const& _data, int _w = 2, HexPrefix _prefix = HexPrefix::DontAdd) { std::ostringstream ret; unsigned ii = 0; @@ -99,27 +99,27 @@ bytes asNibbles(bytesConstRef const& _s); /// Converts a templated integer value to the big-endian byte-stream represented on a templated collection. /// The size of the collection object will be unchanged. If it is too small, it will not represent the /// value properly, if too big then the additional elements will be zeroed out. -/// @a _Out will typically be either std::string or bytes. -/// @a _T will typically by unsigned, u160, u256 or bigint. -template -inline void toBigEndian(_T _val, _Out& o_out) +/// @a Out will typically be either std::string or bytes. +/// @a T will typically by unsigned, u160, u256 or bigint. +template +inline void toBigEndian(T _val, Out& o_out) { for (auto i = o_out.size(); i != 0; _val >>= 8, i--) { - _T v = _val & (_T)0xff; - o_out[i - 1] = (typename _Out::value_type)(uint8_t)v; + T v = _val & (T)0xff; + o_out[i - 1] = (typename Out::value_type)(uint8_t)v; } } /// Converts a big-endian byte-stream represented on a templated collection to a templated integer value. /// @a _In will typically be either std::string or bytes. -/// @a _T will typically by unsigned, u160, u256 or bigint. -template -inline _T fromBigEndian(_In const& _bytes) +/// @a T will typically by unsigned, u160, u256 or bigint. +template +inline T fromBigEndian(_In const& _bytes) { - _T ret = (_T)0; + T ret = (T)0; for (auto i: _bytes) - ret = (_T)((ret << 8) | (byte)(typename std::make_unsigned::type)i); + ret = (T)((ret << 8) | (byte)(typename std::make_unsigned::type)i); return ret; } @@ -131,11 +131,11 @@ inline bytes toBigEndian(u160 _val) { bytes ret(20); toBigEndian(_val, ret); ret /// Convenience function for toBigEndian. /// @returns a byte array just big enough to represent @a _val. -template -inline bytes toCompactBigEndian(_T _val, unsigned _min = 0) +template +inline bytes toCompactBigEndian(T _val, unsigned _min = 0) { int i = 0; - for (_T v = _val; v; ++i, v >>= 8) {} + for (T v = _val; v; ++i, v >>= 8) {} bytes ret(std::max(_min, i), 0); toBigEndian(_val, ret); return ret; @@ -147,11 +147,11 @@ inline bytes toCompactBigEndian(byte _val, unsigned _min = 0) /// Convenience function for toBigEndian. /// @returns a string just big enough to represent @a _val. -template -inline std::string toCompactBigEndianString(_T _val, unsigned _min = 0) +template +inline std::string toCompactBigEndianString(T _val, unsigned _min = 0) { int i = 0; - for (_T v = _val; v; ++i, v >>= 8) {} + for (T v = _val; v; ++i, v >>= 8) {} std::string ret(std::max(_min, i), '\0'); toBigEndian(_val, ret); return ret; @@ -179,8 +179,8 @@ std::string escaped(std::string const& _s, bool _all = true); /// Determines the length of the common prefix of the two collections given. /// @returns the number of elements both @a _t and @a _u share, in order, at the beginning. /// @example commonPrefix("Hello world!", "Hello, world!") == 5 -template -unsigned commonPrefix(_T const& _t, _U const& _u) +template +unsigned commonPrefix(T const& _t, _U const& _u) { unsigned s = std::min(_t.size(), _u.size()); for (unsigned i = 0;; ++i) @@ -196,8 +196,8 @@ std::string randomWord(); // General datatype convenience functions. /// Determine bytes required to encode the given integer value. @returns 0 if @a _i is zero. -template -inline unsigned bytesRequired(_T _i) +template +inline unsigned bytesRequired(T _i) { unsigned i = 0; for (; _i != 0; ++i, _i >>= 8) {} @@ -206,39 +206,39 @@ inline unsigned bytesRequired(_T _i) /// Trims a given number of elements from the front of a collection. /// Only works for POD element types. -template -void trimFront(_T& _t, unsigned _elements) +template +void trimFront(T& _t, unsigned _elements) { - static_assert(std::is_pod::value, ""); + static_assert(std::is_pod::value, ""); memmove(_t.data(), _t.data() + _elements, (_t.size() - _elements) * sizeof(_t[0])); _t.resize(_t.size() - _elements); } /// Pushes an element on to the front of a collection. /// Only works for POD element types. -template -void pushFront(_T& _t, _U _e) +template +void pushFront(T& _t, _U _e) { - static_assert(std::is_pod::value, ""); + static_assert(std::is_pod::value, ""); _t.push_back(_e); memmove(_t.data() + 1, _t.data(), (_t.size() - 1) * sizeof(_e)); _t[0] = _e; } /// Concatenate two vectors of elements of POD types. -template -inline std::vector<_T>& operator+=(std::vector::value, _T>::type>& _a, std::vector<_T> const& _b) +template +inline std::vector& operator+=(std::vector::value, T>::type>& _a, std::vector const& _b) { auto s = _a.size(); _a.resize(_a.size() + _b.size()); - memcpy(_a.data() + s, _b.data(), _b.size() * sizeof(_T)); + memcpy(_a.data() + s, _b.data(), _b.size() * sizeof(T)); return _a; } /// Concatenate two vectors of elements. -template -inline std::vector<_T>& operator+=(std::vector::value, _T>::type>& _a, std::vector<_T> const& _b) +template +inline std::vector& operator+=(std::vector::value, T>::type>& _a, std::vector const& _b) { _a.reserve(_a.size() + _b.size()); for (auto& i: _b) @@ -289,16 +289,16 @@ template std::vector operator+(std::vector _a, U const& } /// Concatenate two vectors of elements. -template -inline std::vector<_T> operator+(std::vector<_T> const& _a, std::vector<_T> const& _b) +template +inline std::vector operator+(std::vector const& _a, std::vector const& _b) { - std::vector<_T> ret(_a); + std::vector ret(_a); return ret += _b; } /// Merge two sets of elements. -template -inline std::set<_T>& operator+=(std::set<_T>& _a, std::set<_T> const& _b) +template +inline std::set& operator+=(std::set& _a, std::set const& _b) { for (auto& i: _b) _a.insert(i); @@ -306,13 +306,28 @@ inline std::set<_T>& operator+=(std::set<_T>& _a, std::set<_T> const& _b) } /// Merge two sets of elements. -template -inline std::set<_T> operator+(std::set<_T> const& _a, std::set<_T> const& _b) +template +inline std::set operator+(std::set const& _a, std::set const& _b) { - std::set<_T> ret(_a); + std::set ret(_a); return ret += _b; } +template +std::unordered_map& operator+=(std::unordered_map& _x, std::unordered_map const& _y) +{ + for (auto const& i: _y) + _x.insert(i); + return _x; +} + +template +std::unordered_map operator+(std::unordered_map const& _x, std::unordered_map const& _y) +{ + std::unordered_map ret(_x); + return ret += _y; +} + /// Make normal string from fixed-length string. std::string toString(string32 const& _s); diff --git a/libethereum/BasicGasPricer.cpp b/libethereum/BasicGasPricer.cpp new file mode 100644 index 000000000..145d23594 --- /dev/null +++ b/libethereum/BasicGasPricer.cpp @@ -0,0 +1,95 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file BasicGasPricer.cpp + * @author Gav Wood + * @date 2015 + */ + +#include +#include "BasicGasPricer.h" +#include "BlockChain.h" +using namespace std; +using namespace dev; +using namespace dev::eth; + +void BasicGasPricer::update(BlockChain const& _bc) +{ + unsigned c = 0; + h256 p = _bc.currentHash(); + m_gasPerBlock = _bc.info(p).gasLimit; + + map dist; + u256 total = 0; + + // make gasPrice versus gasUsed distribution for the last 1000 blocks + while (c < 1000 && p) + { + BlockInfo bi = _bc.info(p); + if (bi.transactionsRoot != EmptyTrie) + { + auto bb = _bc.block(p); + RLP r(bb); + BlockReceipts brs(_bc.receipts(bi.hash())); + size_t i = 0; + for (auto const& tr: r[1]) + { + Transaction tx(tr.data(), CheckTransaction::None); + u256 gu = brs.receipts[i].gasUsed(); + dist[tx.gasPrice()] += gu; + total += gu; + i++; + } + } + p = bi.parentHash; + ++c; + } + + // fill m_octiles with weighted gasPrices + if (total > 0) + { + m_octiles[0] = dist.begin()->first; + + // calc mean + u256 mean = 0; + for (auto const& i: dist) + mean += i.first * i.second; + mean /= total; + + // calc standard deviation + u256 sdSquared = 0; + for (auto const& i: dist) + sdSquared += i.second * (i.first - mean) * (i.first - mean); + sdSquared /= total; + + if (sdSquared) + { + long double sd = sqrt(sdSquared.convert_to()); + long double normalizedSd = sd / mean.convert_to(); + + // calc octiles normalized to gaussian distribution + boost::math::normal gauss(1.0, (normalizedSd > 0.01) ? normalizedSd : 0.01); + for (size_t i = 1; i < 8; i++) + m_octiles[i] = u256(mean.convert_to() * boost::math::quantile(gauss, i / 8.0)); + m_octiles[8] = dist.rbegin()->first; + } + else + { + for (size_t i = 0; i < 9; i++) + m_octiles[i] = (i + 1) * mean / 5; + } + } +} diff --git a/libethereum/BasicGasPricer.h b/libethereum/BasicGasPricer.h new file mode 100644 index 000000000..77d4547c5 --- /dev/null +++ b/libethereum/BasicGasPricer.h @@ -0,0 +1,53 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file BasicGasPricer.h + * @author Gav Wood + * @date 2014 + */ + +#pragma once + +#include +#include "GasPricer.h" + +namespace dev +{ +namespace eth +{ + +class BasicGasPricer: public GasPricer +{ +public: + explicit BasicGasPricer(u256 _weiPerRef, u256 _refsPerBlock): m_weiPerRef(_weiPerRef), m_refsPerBlock(_refsPerBlock) {} + + void setRefPrice(u256 _weiPerRef) { if ((bigint)m_refsPerBlock * _weiPerRef > std::numeric_limits::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_weiPerRef = _weiPerRef; } + void setRefBlockFees(u256 _refsPerBlock) { if ((bigint)m_weiPerRef * _refsPerBlock > std::numeric_limits::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_refsPerBlock = _refsPerBlock; } + + u256 ask(State const&) const override { return m_weiPerRef * m_refsPerBlock / m_gasPerBlock; } + u256 bid(TransactionPriority _p = TransactionPriority::Medium) const override { return m_octiles[(int)_p] > 0 ? m_octiles[(int)_p] : (m_weiPerRef * m_refsPerBlock / m_gasPerBlock); } + + void update(BlockChain const& _bc) override; + +private: + u256 m_weiPerRef; + u256 m_refsPerBlock; + u256 m_gasPerBlock = 3141592; + std::array m_octiles; +}; + +} +} diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index 47725e088..4e0266b5b 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -127,7 +127,7 @@ static const unsigned c_minCacheSize = 1024 * 1024 * 32; #endif -BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, WithExisting _we, ProgressCallback const& _p) +BlockChain::BlockChain(bytes const& _genesisBlock, std::string const& _path, WithExisting _we, ProgressCallback const& _p) { // initialise deathrow. m_cacheUsage.resize(c_collectionQueueSize); @@ -137,8 +137,7 @@ BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, WithExisti m_genesisBlock = _genesisBlock; m_genesisHash = sha3(RLP(m_genesisBlock)[0].data()); - open(_path, _we); - if (_we == WithExisting::Verify) + if (open(_path, _we) != c_minorProtocolVersion) rebuild(_path, _p); } @@ -147,24 +146,41 @@ BlockChain::~BlockChain() close(); } -void BlockChain::open(std::string const& _path, WithExisting _we) +unsigned BlockChain::open(std::string const& _path, WithExisting _we) { - std::string path = _path.empty() ? Defaults::get()->m_dbPath : _path; - boost::filesystem::create_directories(path); + string path = _path.empty() ? Defaults::get()->m_dbPath : _path; + string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4)); + string extrasPath = chainPath + "/" + toString(c_databaseVersion); + + boost::filesystem::create_directories(extrasPath); + + bytes status = contents(extrasPath + "/minor"); + unsigned lastMinor = c_minorProtocolVersion; + DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(status)); + if (c_minorProtocolVersion != lastMinor) + { + cnote << "Killing extras database (DB minor version:" << lastMinor << " != our miner version: " << c_minorProtocolVersion << ")."; + DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove_all(extrasPath + "/details.old")); + boost::filesystem::rename(extrasPath + "/extras", extrasPath + "/extras.old"); + boost::filesystem::remove_all(extrasPath + "/state"); + writeFile(extrasPath + "/minor", rlp(c_minorProtocolVersion)); + lastMinor = (unsigned)RLP(status); + } if (_we == WithExisting::Kill) { - boost::filesystem::remove_all(path + "/blocks"); - boost::filesystem::remove_all(path + "/details"); + cnote << "Killing blockchain & extras database (WithExisting::Kill)."; + boost::filesystem::remove_all(chainPath + "/blocks"); + boost::filesystem::remove_all(extrasPath + "/extras"); } ldb::Options o; o.create_if_missing = true; o.max_open_files = 256; - ldb::DB::Open(o, path + "/blocks", &m_blocksDB); - ldb::DB::Open(o, path + "/details", &m_extrasDB); + ldb::DB::Open(o, chainPath + "/blocks", &m_blocksDB); + ldb::DB::Open(o, extrasPath + "/extras", &m_extrasDB); if (!m_blocksDB || !m_extrasDB) { - if (boost::filesystem::space(path + "/blocks").available < 1024) + if (boost::filesystem::space(chainPath + "/blocks").available < 1024) { cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing."; BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace()); @@ -194,7 +210,8 @@ void BlockChain::open(std::string const& _path, WithExisting _we) m_lastBlockHash = l.empty() ? m_genesisHash : *(h256*)l.data(); m_lastBlockNumber = number(m_lastBlockHash); - cnote << "Opened blockchain DB. Latest: " << currentHash(); + cnote << "Opened blockchain DB. Latest: " << currentHash() << (lastMinor == c_minorProtocolVersion ? "(rebuild not needed)" : "*** REBUILD NEEDED ***"); + return lastMinor; } void BlockChain::close() @@ -208,11 +225,11 @@ void BlockChain::close() m_blocks.clear(); } -#define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {} - void BlockChain::rebuild(std::string const& _path, std::function const& _progress, bool _prepPoW) { - std::string path = _path.empty() ? Defaults::get()->m_dbPath : _path; + string path = _path.empty() ? Defaults::get()->m_dbPath : _path; + string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4)); + string extrasPath = chainPath + "/" + toString(c_databaseVersion); #if ETH_PROFILING_GPERF ProfilerStart("BlockChain_rebuild.log"); @@ -220,16 +237,21 @@ void BlockChain::rebuild(std::string const& _path, std::function; class BlockChain { public: - BlockChain(bytes const& _genesisBlock, std::string _path, WithExisting _we, ProgressCallback const& _p = ProgressCallback()); + BlockChain(bytes const& _genesisBlock, std::string const& _path, WithExisting _we = WithExisting::Trust, ProgressCallback const& _p = ProgressCallback()); ~BlockChain(); /// Attempt a database re-open. @@ -261,7 +261,7 @@ public: private: static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); } - void open(std::string const& _path, WithExisting _we = WithExisting::Trust); + unsigned open(std::string const& _path, WithExisting _we = WithExisting::Trust); void close(); template T queryExtras(h256 const& _h, std::unordered_map& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 172a45983..e812fb4e9 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #if ETH_JSONRPC || !ETH_TRUE #include #include @@ -38,53 +37,82 @@ #include "Defaults.h" #include "Executive.h" #include "EthereumHost.h" +#include "Utility.h" using namespace std; using namespace dev; using namespace dev::eth; using namespace p2p; -VersionChecker::VersionChecker(string const& _dbPath): - m_path(_dbPath.size() ? _dbPath : Defaults::dbPath()) +std::ostream& dev::eth::operator<<(std::ostream& _out, ActivityReport const& _r) { - bytes statusBytes = contents(m_path + "/status"); - RLP status(statusBytes); - try - { - auto protocolVersion = (unsigned)status[0]; - (void)protocolVersion; - auto minorProtocolVersion = (unsigned)status[1]; - auto databaseVersion = (unsigned)status[2]; - h256 ourGenesisHash = CanonBlockChain::genesis().hash(); - auto genesisHash = status.itemCount() > 3 ? (h256)status[3] : ourGenesisHash; - - m_action = - databaseVersion != c_databaseVersion || genesisHash != ourGenesisHash ? - WithExisting::Kill - : minorProtocolVersion != eth::c_minorProtocolVersion ? - WithExisting::Verify - : - WithExisting::Trust; - } - catch (...) - { - m_action = WithExisting::Kill; - } + _out << "Since " << toString(_r.since) << " (" << std::chrono::duration_cast(std::chrono::system_clock::now() - _r.since).count(); + _out << "): " << _r.ticks << "ticks"; + return _out; } -void VersionChecker::setOk() +#ifdef _WIN32 +const char* ClientNote::name() { return EthTeal "^" EthBlue " i"; } +const char* ClientChat::name() { return EthTeal "^" EthWhite " o"; } +const char* ClientTrace::name() { return EthTeal "^" EthGray " O"; } +const char* ClientDetail::name() { return EthTeal "^" EthCoal " 0"; } +#else +const char* ClientNote::name() { return EthTeal "⧫" EthBlue " ℹ"; } +const char* ClientChat::name() { return EthTeal "⧫" EthWhite " ◌"; } +const char* ClientTrace::name() { return EthTeal "⧫" EthGray " ◎"; } +const char* ClientDetail::name() { return EthTeal "⧫" EthCoal " ●"; } +#endif + +static const Addresses c_canaries = { - if (m_action != WithExisting::Trust) - { - try - { - boost::filesystem::create_directory(m_path); - } - catch (...) - { - cwarn << "Unhandled exception! Failed to create directory: " << m_path << "\n" << boost::current_exception_diagnostic_information(); - } - writeFile(m_path + "/status", rlpList(eth::c_protocolVersion, eth::c_minorProtocolVersion, c_databaseVersion, CanonBlockChain::genesis().hash())); - } + Address("4bb7e8ae99b645c2b7860b8f3a2328aae28bd80a"), // gav + Address("1baf27b88c48dd02b744999cf3522766929d2b2a"), // vitalik + Address("a8edb1ac2c86d3d9d78f96cd18001f60df29e52c"), // jeff + Address("60d11b58744784dc97f878f7e3749c0f1381a004") // christoph +}; + +VersionChecker::VersionChecker(string const& _dbPath) +{ + upgradeDatabase(_dbPath); +} + +Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId): + Client(_extNet, make_shared(), _dbPath, _forceAction, _networkId) +{ + startWorking(); +} + +Client::Client(p2p::Host* _extNet, std::shared_ptr _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId): + Worker("eth", 0), + m_vc(_dbPath), + m_bc(_dbPath, _forceAction, [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), + m_gp(_gp), + m_stateDB(State::openDB(_dbPath, _forceAction)), + m_preMine(m_stateDB, BaseState::CanonGenesis), + m_postMine(m_stateDB) +{ + m_lastGetWork = std::chrono::system_clock::now() - chrono::seconds(30); + m_tqReady = m_tq.onReady([=](){ this->onTransactionQueueReady(); }); // TODO: should read m_tq->onReady(thisThread, syncTransactionQueue); + m_bqReady = m_bq.onReady([=](){ this->onBlockQueueReady(); }); // TODO: should read m_bq->onReady(thisThread, syncBlockQueue); + m_bq.setOnBad([=](Exception& ex){ this->onBadBlock(ex); }); + m_bc.setOnBad([=](Exception& ex){ this->onBadBlock(ex); }); + m_farm.onSolutionFound([=](ProofOfWork::Solution const& s){ return this->submitWork(s); }); + + m_gp->update(m_bc); + + auto host = _extNet->registerCapability(new EthereumHost(m_bc, m_tq, m_bq, _networkId)); + m_host = host; + _extNet->addCapability(host, EthereumHost::staticName(), EthereumHost::c_oldProtocolVersion); //TODO: remove this one v61+ protocol is common + + if (_dbPath.size()) + Defaults::setDBPath(_dbPath); + doWork(); + + startWorking(); +} + +Client::~Client() +{ + stopWorking(); } ImportResult Client::queueBlock(bytes const& _block, bool _isSafe) @@ -211,142 +239,6 @@ void Client::onBadBlock(Exception& _ex) const #endif } -void BasicGasPricer::update(BlockChain const& _bc) -{ - unsigned c = 0; - h256 p = _bc.currentHash(); - m_gasPerBlock = _bc.info(p).gasLimit; - - map dist; - u256 total = 0; - - // make gasPrice versus gasUsed distribution for the last 1000 blocks - while (c < 1000 && p) - { - BlockInfo bi = _bc.info(p); - if (bi.transactionsRoot != EmptyTrie) - { - auto bb = _bc.block(p); - RLP r(bb); - BlockReceipts brs(_bc.receipts(bi.hash())); - size_t i = 0; - for (auto const& tr: r[1]) - { - Transaction tx(tr.data(), CheckTransaction::None); - u256 gu = brs.receipts[i].gasUsed(); - dist[tx.gasPrice()] += gu; - total += gu; - i++; - } - } - p = bi.parentHash; - ++c; - } - - // fill m_octiles with weighted gasPrices - if (total > 0) - { - m_octiles[0] = dist.begin()->first; - - // calc mean - u256 mean = 0; - for (auto const& i: dist) - mean += i.first * i.second; - mean /= total; - - // calc standard deviation - u256 sdSquared = 0; - for (auto const& i: dist) - sdSquared += i.second * (i.first - mean) * (i.first - mean); - sdSquared /= total; - - if (sdSquared) - { - long double sd = sqrt(sdSquared.convert_to()); - long double normalizedSd = sd / mean.convert_to(); - - // calc octiles normalized to gaussian distribution - boost::math::normal gauss(1.0, (normalizedSd > 0.01) ? normalizedSd : 0.01); - for (size_t i = 1; i < 8; i++) - m_octiles[i] = u256(mean.convert_to() * boost::math::quantile(gauss, i / 8.0)); - m_octiles[8] = dist.rbegin()->first; - } - else - { - for (size_t i = 0; i < 9; i++) - m_octiles[i] = (i + 1) * mean / 5; - } - } -} - -std::ostream& dev::eth::operator<<(std::ostream& _out, ActivityReport const& _r) -{ - _out << "Since " << toString(_r.since) << " (" << std::chrono::duration_cast(std::chrono::system_clock::now() - _r.since).count(); - _out << "): " << _r.ticks << "ticks"; - return _out; -} - -#ifdef _WIN32 -const char* ClientNote::name() { return EthTeal "^" EthBlue " i"; } -const char* ClientChat::name() { return EthTeal "^" EthWhite " o"; } -const char* ClientTrace::name() { return EthTeal "^" EthGray " O"; } -const char* ClientDetail::name() { return EthTeal "^" EthCoal " 0"; } -#else -const char* ClientNote::name() { return EthTeal "⧫" EthBlue " ℹ"; } -const char* ClientChat::name() { return EthTeal "⧫" EthWhite " ◌"; } -const char* ClientTrace::name() { return EthTeal "⧫" EthGray " ◎"; } -const char* ClientDetail::name() { return EthTeal "⧫" EthCoal " ●"; } -#endif - -Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId): - Client(_extNet, make_shared(), _dbPath, _forceAction, _networkId) -{ - startWorking(); -} - -Client::Client(p2p::Host* _extNet, std::shared_ptr _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId): - Worker("eth", 0), - m_vc(_dbPath), - m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }), - m_gp(_gp), - m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), - m_preMine(m_stateDB, BaseState::CanonGenesis), - m_postMine(m_stateDB) -{ - m_lastGetWork = std::chrono::system_clock::now() - chrono::seconds(30); - m_tqReady = m_tq.onReady([=](){ this->onTransactionQueueReady(); }); // TODO: should read m_tq->onReady(thisThread, syncTransactionQueue); - m_bqReady = m_bq.onReady([=](){ this->onBlockQueueReady(); }); // TODO: should read m_bq->onReady(thisThread, syncBlockQueue); - m_bq.setOnBad([=](Exception& ex){ this->onBadBlock(ex); }); - m_bc.setOnBad([=](Exception& ex){ this->onBadBlock(ex); }); - m_farm.onSolutionFound([=](ProofOfWork::Solution const& s){ return this->submitWork(s); }); - - m_gp->update(m_bc); - - auto host = _extNet->registerCapability(new EthereumHost(m_bc, m_tq, m_bq, _networkId)); - m_host = host; - _extNet->addCapability(host, EthereumHost::staticName(), EthereumHost::c_oldProtocolVersion); //TODO: remove this one v61+ protocol is common - - if (_dbPath.size()) - Defaults::setDBPath(_dbPath); - m_vc.setOk(); - doWork(); - - startWorking(); -} - -Client::~Client() -{ - stopWorking(); -} - -static const Addresses c_canaries = -{ - Address("4bb7e8ae99b645c2b7860b8f3a2328aae28bd80a"), // gav - Address("1baf27b88c48dd02b744999cf3522766929d2b2a"), // vitalik - Address("a8edb1ac2c86d3d9d78f96cd18001f60df29e52c"), // jeff - Address("60d11b58744784dc97f878f7e3749c0f1381a004") // christoph -}; - bool Client::isChainBad() const { unsigned numberBad = 0; diff --git a/libethereum/Client.h b/libethereum/Client.h index 8deeb3353..343ca5b60 100644 --- a/libethereum/Client.h +++ b/libethereum/Client.h @@ -64,33 +64,6 @@ class VersionChecker { public: VersionChecker(std::string const& _dbPath); - - void setOk(); - WithExisting action() const { return m_action; } - -private: - WithExisting m_action; - std::string m_path; -}; - -class BasicGasPricer: public GasPricer -{ -public: - explicit BasicGasPricer(u256 _weiPerRef, u256 _refsPerBlock): m_weiPerRef(_weiPerRef), m_refsPerBlock(_refsPerBlock) {} - - void setRefPrice(u256 _weiPerRef) { if ((bigint)m_refsPerBlock * _weiPerRef > std::numeric_limits::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_weiPerRef = _weiPerRef; } - void setRefBlockFees(u256 _refsPerBlock) { if ((bigint)m_weiPerRef * _refsPerBlock > std::numeric_limits::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_refsPerBlock = _refsPerBlock; } - - u256 ask(State const&) const override { return m_weiPerRef * m_refsPerBlock / m_gasPerBlock; } - u256 bid(TransactionPriority _p = TransactionPriority::Medium) const override { return m_octiles[(int)_p] > 0 ? m_octiles[(int)_p] : (m_weiPerRef * m_refsPerBlock / m_gasPerBlock); } - - void update(BlockChain const& _bc) override; - -private: - u256 m_weiPerRef; - u256 m_refsPerBlock; - u256 m_gasPerBlock = 3141592; - std::array m_octiles; }; struct ClientNote: public LogChannel { static const char* name(); static const int verbosity = 2; }; diff --git a/libethereum/EthereumHost.cpp b/libethereum/EthereumHost.cpp index 9980f4339..55a1d1bf0 100644 --- a/libethereum/EthereumHost.cpp +++ b/libethereum/EthereumHost.cpp @@ -286,7 +286,7 @@ void EthereumHost::onPeerTransactions(std::shared_ptr _peer, RLP c unsigned itemCount = _r.itemCount(); clog(NetAllDetail) << "Transactions (" << dec << itemCount << "entries)"; Guard l(_peer->x_knownTransactions); - for (unsigned i = 0; i < min(itemCount, 256); ++i) // process 256 transactions at most. TODO: much better solution. + for (unsigned i = 0; i < min(itemCount, 32); ++i) // process 256 transactions at most. TODO: much better solution. { auto h = sha3(_r[i].data()); _peer->m_knownTransactions.insert(h); diff --git a/libethereum/GasPricer.cpp b/libethereum/GasPricer.cpp new file mode 100644 index 000000000..21f061afc --- /dev/null +++ b/libethereum/GasPricer.cpp @@ -0,0 +1,26 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file GasPricer.cpp + * @author Gav Wood + * @date 2015 + */ + +#include "GasPricer.h" + +using namespace std; +using namespace dev; +using namespace dev::eth; diff --git a/libethereum/GasPricer.h b/libethereum/GasPricer.h new file mode 100644 index 000000000..f0fc6b520 --- /dev/null +++ b/libethereum/GasPricer.h @@ -0,0 +1,74 @@ +/* + This file is part of cpp-ethereum. + + cpp-ethereum is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + cpp-ethereum is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with cpp-ethereum. If not, see . +*/ +/** @file GasPricer.h + * @author Gav Wood + * @date 2014 + */ + +#pragma once + +#include + +namespace dev +{ +namespace eth +{ + +class State; +class BlockChain; + +enum class TransactionPriority +{ + Lowest = 0, + Low = 2, + Medium = 4, + High = 6, + Highest = 8 +}; + +class GasPricer +{ +public: + GasPricer() = default; + virtual ~GasPricer() = default; + + virtual u256 ask(State const&) const = 0; + virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0; + + virtual void update(BlockChain const&) {} +}; + +class TrivialGasPricer: public GasPricer +{ +public: + TrivialGasPricer() = default; + TrivialGasPricer(u256 const& _ask, u256 const& _bid): m_ask(_ask), m_bid(_bid) {} + + void setAsk(u256 const& _ask) { m_ask = _ask; } + void setBid(u256 const& _bid) { m_bid = _bid; } + + u256 ask() const { return m_ask; } + u256 ask(State const&) const override { return m_ask; } + u256 bid(TransactionPriority = TransactionPriority::Medium) const override { return m_bid; } + +private: + u256 m_ask = 10 * szabo; + u256 m_bid = 10 * szabo; +}; + +} +} diff --git a/libethereum/State.cpp b/libethereum/State.cpp index 3f9978569..6c8927313 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -41,6 +41,7 @@ using namespace std; using namespace dev; using namespace dev::eth; +namespace fs = boost::filesystem; #define ctrace clog(StateTrace) #define ETH_TIMED_ENACTMENTS 0 @@ -52,23 +53,27 @@ const char* StateDetail::name() { return EthViolet "⚙" EthWhite " ◌"; } const char* StateTrace::name() { return EthViolet "⚙" EthGray " ◎"; } const char* StateChat::name() { return EthViolet "⚙" EthWhite " ◌"; } -OverlayDB State::openDB(std::string _path, WithExisting _we) +OverlayDB State::openDB(std::string const& _basePath, WithExisting _we) { - if (_path.empty()) - _path = Defaults::get()->m_dbPath; - boost::filesystem::create_directory(_path); + std::string path = _basePath.empty() ? Defaults::get()->m_dbPath : _basePath; if (_we == WithExisting::Kill) - boost::filesystem::remove_all(_path + "/state"); + { + cnote << "Killing state database (WithExisting::Kill)."; + boost::filesystem::remove_all(path + "/state"); + } + + path += "/" + toHex(CanonBlockChain::genesis().hash().ref().cropped(0, 4)) + "/" + toString(c_databaseVersion); + boost::filesystem::create_directory(path); ldb::Options o; o.max_open_files = 256; o.create_if_missing = true; ldb::DB* db = nullptr; - ldb::DB::Open(o, _path + "/state", &db); + ldb::DB::Open(o, path + "/state", &db); if (!db) { - if (boost::filesystem::space(_path + "/state").available < 1024) + if (boost::filesystem::space(path + "/state").available < 1024) { cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing."; BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace()); diff --git a/libethereum/State.h b/libethereum/State.h index 6c54cedf7..2a63aeda4 100644 --- a/libethereum/State.h +++ b/libethereum/State.h @@ -37,6 +37,7 @@ #include "Transaction.h" #include "TransactionReceipt.h" #include "AccountDiff.h" +#include "GasPricer.h" namespace dev { @@ -80,45 +81,6 @@ enum class BaseState CanonGenesis }; -enum class TransactionPriority -{ - Lowest = 0, - Low = 2, - Medium = 4, - High = 6, - Highest = 8 -}; - -class GasPricer -{ -public: - GasPricer() = default; - virtual ~GasPricer() = default; - - virtual u256 ask(State const&) const = 0; - virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0; - - virtual void update(BlockChain const&) {} -}; - -class TrivialGasPricer: public GasPricer -{ -public: - TrivialGasPricer() = default; - TrivialGasPricer(u256 const& _ask, u256 const& _bid): m_ask(_ask), m_bid(_bid) {} - - void setAsk(u256 const& _ask) { m_ask = _ask; } - void setBid(u256 const& _bid) { m_bid = _bid; } - - u256 ask() const { return m_ask; } - u256 ask(State const&) const override { return m_ask; } - u256 bid(TransactionPriority = TransactionPriority::Medium) const override { return m_bid; } - -private: - u256 m_ask = 10 * szabo; - u256 m_bid = 10 * szabo; -}; - enum class Permanence { Reverted, @@ -171,7 +133,7 @@ public: Address address() const { return m_ourAddress; } /// Open a DB - useful for passing into the constructor & keeping for other states that are necessary. - static OverlayDB openDB(std::string _path, WithExisting _we = WithExisting::Trust); + static OverlayDB openDB(std::string const& _path, WithExisting _we = WithExisting::Trust); static OverlayDB openDB(WithExisting _we = WithExisting::Trust) { return openDB(std::string(), _we); } OverlayDB const& db() const { return m_db; } OverlayDB& db() { return m_db; } diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 8fcf3cfb6..8b0451d65 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -98,11 +98,7 @@ ImportResult TransactionQueue::import(Transaction const& _transaction, ImportCal std::unordered_map TransactionQueue::transactions() const { ReadGuard l(m_lock); - auto ret = m_current; - for (auto const& i: m_future) - if (i.second.nonce() < maxNonce_WITH_LOCK(i.second.sender())) - ret.insert(i); - return ret; + return m_current + m_future; } ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb) @@ -113,6 +109,9 @@ ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transactio // If it doesn't work, the signature is bad. // The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction). + + // Remove any prior transaction with the same nonce but a lower gas price. + // Bomb out if there's a prior transaction with higher gas price. auto r = m_senders.equal_range(_transaction.from()); for (auto it = r.first; it != r.second; ++it) if (m_current.count(it->second) && m_current[it->second].nonce() == _transaction.nonce()) diff --git a/libethereum/Utility.cpp b/libethereum/Utility.cpp index adfea4a51..a9158a363 100644 --- a/libethereum/Utility.cpp +++ b/libethereum/Utility.cpp @@ -22,11 +22,16 @@ #include "Utility.h" #include -#include +#include #include +#include +#include +#include +#include "Defaults.h" using namespace std; using namespace dev; using namespace dev::eth; +namespace fs = boost::filesystem; bytes dev::eth::parseData(string const& _args) { @@ -84,3 +89,47 @@ bytes dev::eth::parseData(string const& _args) return m_data; } + +void dev::eth::upgradeDatabase(std::string const& _basePath) +{ + std::string path = _basePath.empty() ? Defaults::get()->dbPath() : _basePath; + + if (fs::exists(path + "/state") && fs::exists(path + "/details") && fs::exists(path + "/blocks")) + { + // upgrade + cnote << "Upgrading database to new layout..."; + bytes statusBytes = contents(path + "/status"); + RLP status(statusBytes); + try + { + auto minorProtocolVersion = (unsigned)status[1]; + auto databaseVersion = (unsigned)status[2]; + auto genesisHash = (h256)status[3]; + + string chainPath = path + "/" + toHex(genesisHash.ref().cropped(0, 4)); + string extrasPath = chainPath + "/" + toString(databaseVersion); + + // write status + if (!fs::exists(chainPath + "/blocks")) + { + boost::filesystem::create_directories(chainPath); + fs::rename(path + "/blocks", chainPath + "/blocks"); + + if (!fs::exists(extrasPath + "/extras")) + { + boost::filesystem::create_directories(extrasPath); + fs::rename(path + "/details", extrasPath + "/extras"); + fs::rename(path + "/state", extrasPath + "/state"); + writeFile(extrasPath + "/minor", rlp(minorProtocolVersion)); + + fs::remove_all(path + "/status"); + } + } + } + catch (...) + { + cwarn << "Couldn't upgrade - bad status"; + } + } +} + diff --git a/libethereum/Utility.h b/libethereum/Utility.h index 893604139..0dfe8509b 100644 --- a/libethereum/Utility.h +++ b/libethereum/Utility.h @@ -42,5 +42,7 @@ namespace eth */ bytes parseData(std::string const& _args); +void upgradeDatabase(std::string const& _basePath); + } }