diff --git a/alethzero/MainWin.cpp b/alethzero/MainWin.cpp index 4963d2742..ff505d5f2 100644 --- a/alethzero/MainWin.cpp +++ b/alethzero/MainWin.cpp @@ -1097,30 +1097,33 @@ void Main::refreshBlockChain() blockItem->setSelected(true); int n = 0; - auto b = bc.block(h); - for (auto const& i: RLP(b)[1]) - { - Transaction t(i.data(), CheckTransaction::Everything); - QString s = t.receiveAddress() ? - QString(" %2 %5> %3: %1 [%4]") - .arg(formatBalance(t.value()).c_str()) - .arg(render(t.safeSender())) - .arg(render(t.receiveAddress())) - .arg((unsigned)t.nonce()) - .arg(ethereum()->codeAt(t.receiveAddress()).size() ? '*' : '-') : - QString(" %2 +> %3: %1 [%4]") - .arg(formatBalance(t.value()).c_str()) - .arg(render(t.safeSender())) - .arg(render(right160(sha3(rlpList(t.safeSender(), t.nonce()))))) - .arg((unsigned)t.nonce()); - QListWidgetItem* txItem = new QListWidgetItem(s, ui->blocks); - auto hba = QByteArray((char const*)h.data(), h.size); - txItem->setData(Qt::UserRole, hba); - txItem->setData(Qt::UserRole + 1, n); - if (oldSelected == hba) - txItem->setSelected(true); - n++; + try { + auto b = bc.block(h); + for (auto const& i: RLP(b)[1]) + { + Transaction t(i.data(), CheckTransaction::Everything); + QString s = t.receiveAddress() ? + QString(" %2 %5> %3: %1 [%4]") + .arg(formatBalance(t.value()).c_str()) + .arg(render(t.safeSender())) + .arg(render(t.receiveAddress())) + .arg((unsigned)t.nonce()) + .arg(ethereum()->codeAt(t.receiveAddress()).size() ? '*' : '-') : + QString(" %2 +> %3: %1 [%4]") + .arg(formatBalance(t.value()).c_str()) + .arg(render(t.safeSender())) + .arg(render(right160(sha3(rlpList(t.safeSender(), t.nonce()))))) + .arg((unsigned)t.nonce()); + QListWidgetItem* txItem = new QListWidgetItem(s, ui->blocks); + auto hba = QByteArray((char const*)h.data(), h.size); + txItem->setData(Qt::UserRole, hba); + txItem->setData(Qt::UserRole + 1, n); + if (oldSelected == hba) + txItem->setSelected(true); + n++; + } } + catch (...) {} }; if (filters.empty()) diff --git a/libdevcore/Common.cpp b/libdevcore/Common.cpp index f27637dec..b3d70c538 100644 --- a/libdevcore/Common.cpp +++ b/libdevcore/Common.cpp @@ -27,7 +27,7 @@ using namespace dev; namespace dev { -char const* Version = "0.9.13"; +char const* Version = "0.9.14"; } diff --git a/libdevcrypto/MemoryDB.cpp b/libdevcrypto/MemoryDB.cpp index a207bb7d6..907e6abe6 100644 --- a/libdevcrypto/MemoryDB.cpp +++ b/libdevcrypto/MemoryDB.cpp @@ -32,51 +32,55 @@ const char* DBWarn::name() { return "TDB"; } std::map MemoryDB::get() const { - if (!m_enforceRefs) - return m_over; std::map ret; - for (auto const& i: m_refCount) - if (i.second) - ret.insert(*m_over.find(i.first)); + for (auto const& i: m_main) + if (!m_enforceRefs || i.second.second > 0) + ret.insert(make_pair(i.first, i.second.first)); return ret; } -std::string MemoryDB::lookup(h256 _h) const +std::string MemoryDB::lookup(h256 const& _h) const { - auto it = m_over.find(_h); - if (it != m_over.end()) + auto it = m_main.find(_h); + if (it != m_main.end()) { - if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first))) - return it->second; + if (!m_enforceRefs || it->second.second > 0) + return it->second.first; // else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first)) // cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h; } return std::string(); } -bool MemoryDB::exists(h256 _h) const +bool MemoryDB::exists(h256 const& _h) const { - auto it = m_over.find(_h); - if (it != m_over.end() && (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))) + auto it = m_main.find(_h); + if (it != m_main.end() && (!m_enforceRefs || it->second.second > 0)) return true; return false; } -void MemoryDB::insert(h256 _h, bytesConstRef _v) +void MemoryDB::insert(h256 const& _h, bytesConstRef _v) { - m_over[_h] = _v.toString(); - m_refCount[_h]++; + auto it = m_main.find(_h); + if (it != m_main.end()) + { + it->second.first = _v.toString(); + it->second.second++; + } + else + m_main[_h] = make_pair(_v.toString(), 1); #if ETH_PARANOIA - dbdebug << "INST" << _h << "=>" << m_refCount[_h]; + dbdebug << "INST" << _h << "=>" << m_main[_h].second; #endif } -bool MemoryDB::kill(h256 _h) +bool MemoryDB::kill(h256 const& _h) { - if (m_refCount.count(_h)) + if (m_main.count(_h)) { - if (m_refCount[_h] > 0) - --m_refCount[_h]; + if (m_main[_h].second > 0) + m_main[_h].second--; #if ETH_PARANOIA else { @@ -85,7 +89,7 @@ bool MemoryDB::kill(h256 _h) dbdebug << "NOKILL-WAS" << _h; return false; } - dbdebug << "KILL" << _h << "=>" << m_refCount[_h]; + dbdebug << "KILL" << _h << "=>" << m_main[_h].second; return true; } else @@ -101,16 +105,18 @@ bool MemoryDB::kill(h256 _h) void MemoryDB::purge() { - for (auto const& i: m_refCount) - if (!i.second) - m_over.erase(i.first); + for (auto it = m_main.begin(); it != m_main.end(); ) + if (it->second.second) + ++it; + else + it = m_main.erase(it); } set MemoryDB::keys() const { set ret; - for (auto const& i: m_refCount) - if (i.second && h128(i.first.ref().cropped(0, 16))) + for (auto const& i: m_main) + if (i.second.second) ret.insert(i.first); return ret; } diff --git a/libdevcrypto/MemoryDB.h b/libdevcrypto/MemoryDB.h index 58b1339a5..71428ecdb 100644 --- a/libdevcrypto/MemoryDB.h +++ b/libdevcrypto/MemoryDB.h @@ -44,28 +44,24 @@ class MemoryDB public: MemoryDB() {} - void clear() { m_over.clear(); } + void clear() { m_main.clear(); } // WARNING !!!! didn't originally clear m_refCount!!! std::map get() const; - std::string lookup(h256 _h) const; - bool exists(h256 _h) const; - void insert(h256 _h, bytesConstRef _v); - bool kill(h256 _h); + std::string lookup(h256 const& _h) const; + bool exists(h256 const& _h) const; + void insert(h256 const& _h, bytesConstRef _v); + bool kill(h256 const& _h); void purge(); - bytes lookupAux(h256 _h) const { auto h = aux(_h); return m_aux.count(h) ? m_aux.at(h) : bytes(); } - void removeAux(h256 _h) { m_auxActive.erase(aux(_h)); } - void insertAux(h256 _h, bytesConstRef _v) { auto h = aux(_h); m_auxActive.insert(h); m_aux[h] = _v.toBytes(); } + bytes lookupAux(h256 const& _h) const { try { return m_aux.at(_h).first; } catch (...) { return bytes(); } } + void removeAux(h256 const& _h) { m_aux[_h].second = false; } + void insertAux(h256 const& _h, bytesConstRef _v) { m_aux[_h] = make_pair(_v.toBytes(), true); } std::set keys() const; protected: - static h256 aux(h256 _k) { return h256(sha3(_k).ref().cropped(0, 24), h256::AlignLeft); } - - std::map m_over; - std::map m_refCount; - std::set m_auxActive; - std::map m_aux; + std::map> m_main; + std::map> m_aux; mutable bool m_enforceRefs = false; }; @@ -83,7 +79,7 @@ private: inline std::ostream& operator<<(std::ostream& _out, MemoryDB const& _m) { - for (auto i: _m.get()) + for (auto const& i: _m.get()) { _out << i.first << ": "; _out << RLP(i.second); diff --git a/libdevcrypto/OverlayDB.cpp b/libdevcrypto/OverlayDB.cpp index e8bd609b0..87c8a0927 100644 --- a/libdevcrypto/OverlayDB.cpp +++ b/libdevcrypto/OverlayDB.cpp @@ -41,23 +41,22 @@ void OverlayDB::commit() { ldb::WriteBatch batch; // cnote << "Committing nodes to disk DB:"; - for (auto const& i: m_over) + for (auto const& i: m_main) { -// cnote << i.first << "#" << m_refCount[i.first]; - if (m_refCount[i.first]) - batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.data(), i.second.size())); +// cnote << i.first << "#" << m_main[i.first].second; + if (i.second.second) + batch.Put(ldb::Slice((char const*)i.first.data(), i.first.size), ldb::Slice(i.second.first.data(), i.second.first.size())); } - for (auto const& i: m_auxActive) - if (m_aux.count(i)) + for (auto const& i: m_aux) + if (i.second.second) { - batch.Put(i.ref(), bytesConstRef(&m_aux[i])); - m_aux.erase(i); + bytes b = i.first.asBytes(); + b.push_back(255); // for aux + batch.Put(bytesConstRef(&b), bytesConstRef(&i.second.first)); } m_db->Write(m_writeOptions, &batch); - m_auxActive.clear(); m_aux.clear(); - m_over.clear(); - m_refCount.clear(); + m_main.clear(); } } @@ -67,7 +66,9 @@ bytes OverlayDB::lookupAux(h256 _h) const if (!ret.empty()) return ret; std::string v; - m_db->Get(m_readOptions, aux(_h).ref(), &v); + bytes b = _h.asBytes(); + b.push_back(255); // for aux + m_db->Get(m_readOptions, bytesConstRef(&b), &v); if (v.empty()) cwarn << "Aux not found: " << _h; return asBytes(v); @@ -75,8 +76,7 @@ bytes OverlayDB::lookupAux(h256 _h) const void OverlayDB::rollback() { - m_over.clear(); - m_refCount.clear(); + m_main.clear(); } std::string OverlayDB::lookup(h256 _h) const diff --git a/libdevcrypto/TrieDB.h b/libdevcrypto/TrieDB.h index a707c30f0..ef628d20b 100644 --- a/libdevcrypto/TrieDB.h +++ b/libdevcrypto/TrieDB.h @@ -47,6 +47,11 @@ struct InvalidTrie: virtual dev::Exception {}; extern const h256 c_shaNull; extern const h256 EmptyTrie; +enum class Verification { + Skip, + Normal +}; + /** * @brief Merkle Patricia Tree "Trie": a modifed base-16 Radix tree. * This version uses a database backend. @@ -69,23 +74,26 @@ public: using DB = _DB; GenericTrieDB(DB* _db = nullptr): m_db(_db) {} - GenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } + GenericTrieDB(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { open(_db, _root, _v); } ~GenericTrieDB() {} void open(DB* _db) { m_db = _db; } - void open(DB* _db, h256 _root) { m_db = _db; setRoot(_root); } + void open(DB* _db, h256 const& _root, Verification _v = Verification::Normal) { m_db = _db; setRoot(_root, _v); } void init() { setRoot(insertNode(&RLPNull)); assert(node(m_root).size()); } - void setRoot(h256 _root) + void setRoot(h256 const& _root, Verification _v = Verification::Normal) { m_root = _root; - if (m_root == c_shaNull && !m_db->exists(m_root)) - init(); + if (_v == Verification::Normal) + { + if (m_root == c_shaNull && !m_db->exists(m_root)) + init(); - /*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/ - if (!node(m_root).size()) - BOOST_THROW_EXCEPTION(RootNotFound()); + /*std::cout << "Setting root to " << _root << " (patched to " << m_root << ")" << std::endl;*/ + if (!node(m_root).size()) + BOOST_THROW_EXCEPTION(RootNotFound()); + } } /// True if the trie is uninitialised (i.e. that the DB doesn't contain the root node). @@ -93,7 +101,7 @@ public: /// True if the trie is initialised but empty (i.e. that the DB contains the root node which is empty). bool isEmpty() const { return m_root == c_shaNull && node(m_root).size(); } - h256 root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; } // patch the root in the case of the empty trie. TODO: handle this properly. + h256 const& root() const { if (!node(m_root).size()) BOOST_THROW_EXCEPTION(BadRoot()); /*std::cout << "Returning root as " << ret << " (really " << m_root << ")" << std::endl;*/ return m_root; } // patch the root in the case of the empty trie. TODO: handle this properly. void debugPrint() {} @@ -301,7 +309,7 @@ public: using KeyType = _KeyType; SpecificTrieDB(DB* _db = nullptr): Generic(_db) {} - SpecificTrieDB(DB* _db, h256 _root): Generic(_db, _root) {} + SpecificTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Generic(_db, _root, _v) {} std::string operator[](KeyType _k) const { return at(_k); } @@ -349,7 +357,7 @@ public: using DB = _DB; HashedGenericTrieDB(DB* _db = nullptr): Super(_db) {} - HashedGenericTrieDB(DB* _db, h256 _root): Super(_db, _root) {} + HashedGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal): Super(_db, _root, _v) {} using Super::open; using Super::init; @@ -402,20 +410,20 @@ class FatGenericTrieDB: public GenericTrieDB public: FatGenericTrieDB(DB* _db): Super(_db), m_secure(_db) {} - FatGenericTrieDB(DB* _db, h256 _root) { open(_db, _root); } + FatGenericTrieDB(DB* _db, h256 _root, Verification _v = Verification::Normal) { open(_db, _root, _v); } - void open(DB* _db, h256 _root) { Super::open(_db); m_secure.open(_db); setRoot(_root); } + void open(DB* _db, h256 _root, Verification _v = Verification::Normal) { Super::open(_db); m_secure.open(_db); setRoot(_root, _v); } void init() { Super::init(); m_secure.init(); syncRoot(); } - void setRoot(h256 _root) + void setRoot(h256 _root, Verification _v = Verification::Normal) { if (!m_secure.isNull()) Super::db()->removeAux(m_secure.root()); - m_secure.setRoot(_root); + m_secure.setRoot(_root, _v); auto rb = Super::db()->lookupAux(m_secure.root()); auto r = h256(rb); - Super::setRoot(r); + Super::setRoot(r, _v); } h256 root() const { return m_secure.root(); } @@ -853,8 +861,8 @@ template bytes GenericTrieDB::mergeAt(RLP const& _orig, NibbleSli template void GenericTrieDB::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v) { -#if ETH_PARANOIA - tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash() : std::string()); +#if ETH_PARANOIA || !ETH_TRUE + tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash().abridged() : std::string()); #endif RLP r = _orig; @@ -1008,8 +1016,8 @@ template bytes GenericTrieDB::deleteAt(RLP const& _orig, NibbleSl template bool GenericTrieDB::deleteAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k) { -#if ETH_PARANOIA - tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash() : std::string()); +#if ETH_PARANOIA || !ETH_TRUE + tdebug << "deleteAtAux " << _orig << _k << sha3(_orig.data()) << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash().abridged() : std::string()); #endif bytes b = _orig.isEmpty() ? bytes() : deleteAt(_orig.isList() ? _orig : RLP(node(_orig.toHash())), _k); diff --git a/libethcore/Common.cpp b/libethcore/Common.cpp index 01c8cfa4c..2c7601dbc 100644 --- a/libethcore/Common.cpp +++ b/libethcore/Common.cpp @@ -36,7 +36,7 @@ namespace eth { const unsigned c_protocolVersion = 60; -const unsigned c_minorProtocolVersion = 1; +const unsigned c_minorProtocolVersion = 2; const unsigned c_databaseBaseVersion = 9; #if ETH_FATDB const unsigned c_databaseVersionModifier = 1; diff --git a/libethereum/BlockChain.cpp b/libethereum/BlockChain.cpp index a4d070171..69078b400 100644 --- a/libethereum/BlockChain.cpp +++ b/libethereum/BlockChain.cpp @@ -47,7 +47,7 @@ using namespace dev::eth; namespace js = json_spirit; #define ETH_CATCH 1 -#define ETH_TIMED_IMPORTS 0 +#define ETH_TIMED_IMPORTS 1 #ifdef _WIN32 const char* BlockChainDebug::name() { return EthBlue "8" EthWhite " <>"; } @@ -303,7 +303,7 @@ LastHashes BlockChain::lastHashes(unsigned _n) const tuple BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max) { - _bq.tick(*this); +// _bq.tick(*this); vector blocks; _bq.drain(blocks, _max); @@ -445,8 +445,8 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import ldb::WriteBatch blocksBatch; ldb::WriteBatch extrasBatch; - h256 newLastBlockHash; - unsigned newLastBlockNumber = 0; + h256 newLastBlockHash = currentHash(); + unsigned newLastBlockNumber = number(); u256 td; #if ETH_CATCH @@ -478,30 +478,27 @@ ImportRoute BlockChain::import(bytes const& _block, OverlayDB const& _db, Import #endif // All ok - insert into DB - { - // ensure parent is cached for later addition. - // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard - // together with an "ensureCachedWithUpdatableLock(l)" method. - // This is safe in practice since the caches don't get flushed nearly often enough to be - // done here. - details(bi.parentHash); - WriteGuard l(x_details); + + // ensure parent is cached for later addition. + // TODO: this is a bit horrible would be better refactored into an enveloping UpgradableGuard + // together with an "ensureCachedWithUpdatableLock(l)" method. + // This is safe in practice since the caches don't get flushed nearly often enough to be + // done here. + details(bi.parentHash); + ETH_WRITE_GUARDED(x_details) m_details[bi.parentHash].children.push_back(bi.hash()); - } #if ETH_TIMED_IMPORTS collation = t.elapsed(); t.restart(); #endif - { - ReadGuard l2(x_details); - extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp())); + blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block)); + ETH_READ_GUARDED(x_details) extrasBatch.Put(toSlice(bi.parentHash, ExtraDetails), (ldb::Slice)dev::ref(m_details[bi.parentHash].rlp())); - extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp())); - extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp())); - blocksBatch.Put(toSlice(bi.hash()), (ldb::Slice)ref(_block)); - } + extrasBatch.Put(toSlice(bi.hash(), ExtraDetails), (ldb::Slice)dev::ref(BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}).rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraLogBlooms), (ldb::Slice)dev::ref(blb.rlp())); + extrasBatch.Put(toSlice(bi.hash(), ExtraReceipts), (ldb::Slice)dev::ref(br.rlp())); #if ETH_TIMED_IMPORTS writing = t.elapsed(); diff --git a/libethereum/BlockQueue.cpp b/libethereum/BlockQueue.cpp index 44ddda637..0cccf8e6b 100644 --- a/libethereum/BlockQueue.cpp +++ b/libethereum/BlockQueue.cpp @@ -74,17 +74,19 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo } UpgradeGuard ul(l); + invariants_WITH_LOCK(); // Check it's not in the future (void)_isOurs; if (bi.timestamp > (u256)time(0)/* && !_isOurs*/) { - m_future.insert(make_pair((unsigned)bi.timestamp, _block.toBytes())); + m_future.insert(make_pair((unsigned)bi.timestamp, make_pair(h, _block.toBytes()))); char buf[24]; time_t bit = (unsigned)bi.timestamp; if (strftime(buf, 24, "%X", localtime(&bit)) == 0) buf[0] = '\0'; // empty if case strftime fails cblockq << "OK - queued for future [" << bi.timestamp << "vs" << time(0) << "] - will wait until" << buf; + invariants_WITH_LOCK(); return ImportResult::FutureTime; } else @@ -94,6 +96,7 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo { m_knownBad.insert(bi.hash()); // bad parent; this is bad too, note it as such + invariants_WITH_LOCK(); return ImportResult::BadChain; } else if (!m_readySet.count(bi.parentHash) && !m_drainingSet.count(bi.parentHash) && !_bc.isKnown(bi.parentHash)) @@ -103,16 +106,18 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo m_unknown.insert(make_pair(bi.parentHash, make_pair(h, _block.toBytes()))); m_unknownSet.insert(h); + invariants_WITH_LOCK(); return ImportResult::UnknownParent; } else { // If valid, append to blocks. cblockq << "OK - ready for chain insertion."; - m_ready.push_back(_block.toBytes()); + m_ready.push_back(make_pair(h, _block.toBytes())); m_readySet.insert(h); + invariants_WITH_LOCK(); - noteReadyWithoutWriteGuard(h); + noteReady_WITH_LOCK(h); m_onReady(); return ImportResult::Success; } @@ -122,27 +127,33 @@ ImportResult BlockQueue::import(bytesConstRef _block, BlockChain const& _bc, boo bool BlockQueue::doneDrain(h256s const& _bad) { WriteGuard l(m_lock); + invariants_WITH_LOCK(); m_drainingSet.clear(); if (_bad.size()) { - vector old; + vector> old; swap(m_ready, old); for (auto& b: old) { - BlockInfo bi(b); + BlockInfo bi(b.second); if (m_knownBad.count(bi.parentHash)) - m_knownBad.insert(bi.hash()); + { + m_knownBad.insert(b.first); + m_readySet.erase(b.first); + } else m_ready.push_back(std::move(b)); } } m_knownBad += _bad; + // GAA!!!! NEVER EMPTY?!?!?! TODO: remove items from readySet! + invariants_WITH_LOCK(); return !m_readySet.empty(); } void BlockQueue::tick(BlockChain const& _bc) { - vector todo; + vector> todo; { UpgradableGuard l(m_lock); if (m_future.empty()) @@ -158,16 +169,18 @@ void BlockQueue::tick(BlockChain const& _bc) { UpgradeGuard l2(l); + invariants_WITH_LOCK(); auto end = m_future.lower_bound(t); for (auto i = m_future.begin(); i != end; ++i) todo.push_back(move(i->second)); m_future.erase(m_future.begin(), end); + invariants_WITH_LOCK(); } } cblockq << "Importing" << todo.size() << "past-future blocks."; for (auto const& b: todo) - import(&b, _bc); + import(&b.second, _bc); } template T advanced(T _t, unsigned _n) @@ -194,25 +207,34 @@ QueueStatus BlockQueue::blockStatus(h256 const& _h) const void BlockQueue::drain(std::vector& o_out, unsigned _max) { WriteGuard l(m_lock); + invariants_WITH_LOCK(); if (m_drainingSet.empty()) { o_out.resize(min(_max, m_ready.size())); for (unsigned i = 0; i < o_out.size(); ++i) - swap(o_out[i], m_ready[i]); + swap(o_out[i], m_ready[i].second); m_ready.erase(m_ready.begin(), advanced(m_ready.begin(), o_out.size())); for (auto const& bs: o_out) { - auto h = sha3(bs); + // TODO: @optimise use map rather than vector & set. + auto h = BlockInfo::headerHash(bs); m_drainingSet.insert(h); m_readySet.erase(h); } // swap(o_out, m_ready); // swap(m_drainingSet, m_readySet); } + invariants_WITH_LOCK(); } -void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) +void BlockQueue::invariants_WITH_LOCK() const { + assert(m_readySet.size() == m_ready.size()); +} + +void BlockQueue::noteReady_WITH_LOCK(h256 const& _good) +{ + invariants_WITH_LOCK(); list goodQueue(1, _good); while (!goodQueue.empty()) { @@ -220,7 +242,7 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) goodQueue.pop_front(); for (auto it = r.first; it != r.second; ++it) { - m_ready.push_back(it->second.second); + m_ready.push_back(it->second); auto newReady = it->second.first; m_unknownSet.erase(newReady); m_readySet.insert(newReady); @@ -228,16 +250,19 @@ void BlockQueue::noteReadyWithoutWriteGuard(h256 _good) } m_unknown.erase(r.first, r.second); } + invariants_WITH_LOCK(); } void BlockQueue::retryAllUnknown() { + invariants_WITH_LOCK(); for (auto it = m_unknown.begin(); it != m_unknown.end(); ++it) { - m_ready.push_back(it->second.second); + m_ready.push_back(it->second); auto newReady = it->second.first; m_unknownSet.erase(newReady); m_readySet.insert(newReady); } m_unknown.clear(); + invariants_WITH_LOCK(); } diff --git a/libethereum/BlockQueue.h b/libethereum/BlockQueue.h index adcc6ab39..1b1612fb7 100644 --- a/libethereum/BlockQueue.h +++ b/libethereum/BlockQueue.h @@ -78,7 +78,7 @@ public: bool doneDrain(h256s const& _knownBad = h256s()); /// Notify the queue that the chain has changed and a new block has attained 'ready' status (i.e. is in the chain). - void noteReady(h256 _b) { WriteGuard l(m_lock); noteReadyWithoutWriteGuard(_b); } + void noteReady(h256 const& _b) { WriteGuard l(m_lock); noteReady_WITH_LOCK(_b); } /// Force a retry of all the blocks with unknown parents. void retryAllUnknown(); @@ -87,7 +87,7 @@ public: std::pair items() const { ReadGuard l(m_lock); return std::make_pair(m_ready.size(), m_unknown.size()); } /// Clear everything. - void clear() { WriteGuard l(m_lock); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); } + void clear() { WriteGuard l(m_lock); invariants_WITH_LOCK(); m_readySet.clear(); m_drainingSet.clear(); m_ready.clear(); m_unknownSet.clear(); m_unknown.clear(); m_future.clear(); invariants_WITH_LOCK(); } /// Return first block with an unknown parent. h256 firstUnknown() const { ReadGuard l(m_lock); return m_unknownSet.size() ? *m_unknownSet.begin() : h256(); } @@ -101,18 +101,18 @@ public: template Handler onReady(T const& _t) { return m_onReady.add(_t); } private: - void noteReadyWithoutWriteGuard(h256 _b); - void notePresentWithoutWriteGuard(bytesConstRef _block); + void noteReady_WITH_LOCK(h256 const& _b); + void invariants_WITH_LOCK() const; mutable boost::shared_mutex m_lock; ///< General lock. - std::set m_readySet; ///< All blocks ready for chain-import. std::set m_drainingSet; ///< All blocks being imported. - std::vector m_ready; ///< List of blocks, in correct order, ready for chain-import. + std::set m_readySet; ///< All blocks ready for chain-import. + std::vector> m_ready; ///< List of blocks, in correct order, ready for chain-import. std::set m_unknownSet; ///< Set of all blocks whose parents are not ready/in-chain. - std::multimap> m_unknown; ///< For transactions that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears. - std::multimap m_future; ///< Set of blocks that are not yet valid. + std::multimap> m_unknown; ///< For blocks that have an unknown parent; we map their parent hash to the block stuff, and insert once the block appears. std::set m_knownBad; ///< Set of blocks that we know will never be valid. - Signal m_onReady; ///< Called when a subsequent call to import transactions will return a non-empty container. Be nice and exit fast. + std::multimap> m_future;///< Set of blocks that are not yet valid. + Signal m_onReady; ///< Called when a subsequent call to import blocks will return a non-empty container. Be nice and exit fast. }; } diff --git a/libethereum/Client.cpp b/libethereum/Client.cpp index 63120cfa3..8d049d404 100644 --- a/libethereum/Client.cpp +++ b/libethereum/Client.cpp @@ -582,11 +582,12 @@ void Client::onChainChanged(ImportRoute const& _ir) m_preMine = newPreMine; DEV_TIMED(working) ETH_WRITE_GUARDED(x_working) m_working = newPreMine; +// Transactions ts = m_postMine.pending(); ETH_READ_GUARDED(x_postMine) for (auto const& t: m_postMine.pending()) { clog(ClientNote) << "Resubmitting post-mine transaction " << t; - m_tq.import(t.rlp(), TransactionQueue::ImportCallback(), IfDropped::Retry); + m_tq.import(t, TransactionQueue::ImportCallback(), IfDropped::Retry); } ETH_READ_GUARDED(x_working) DEV_TIMED(post) ETH_WRITE_GUARDED(x_postMine) m_postMine = m_working; @@ -662,7 +663,7 @@ void Client::doWork() bool t = true; if (m_syncBlockQueue.compare_exchange_strong(t, false)) - syncBlockQueue(); + syncBlockQueue(); // GAAA!!!!! CALLED TOO OFTEN!!! t = true; if (m_syncTransactionQueue.compare_exchange_strong(t, false) && !m_remoteWorking) @@ -735,6 +736,7 @@ eth::State Client::state(unsigned _txi) const ETH_READ_GUARDED(x_postMine) return m_postMine.fromPending(_txi); assert(false); + return State(); } void Client::flushTransactions() diff --git a/libethereum/State.cpp b/libethereum/State.cpp index d5a54985e..83a78e1e8 100644 --- a/libethereum/State.cpp +++ b/libethereum/State.cpp @@ -152,7 +152,7 @@ State::State(OverlayDB const& _db, BlockChain const& _bc, h256 _h): State::State(State const& _s): m_db(_s.m_db), - m_state(&m_db, _s.m_state.root()), + m_state(&m_db, _s.m_state.root(), Verification::Skip), m_transactions(_s.m_transactions), m_receipts(_s.m_receipts), m_transactionSet(_s.m_transactionSet), @@ -184,7 +184,7 @@ void State::paranoia(std::string const& _when, bool _enforceRefs) const State& State::operator=(State const& _s) { m_db = _s.m_db; - m_state.open(&m_db, _s.m_state.root()); + m_state.open(&m_db, _s.m_state.root(), Verification::Skip); m_transactions = _s.m_transactions; m_receipts = _s.m_receipts; m_transactionSet = _s.m_transactionSet; diff --git a/libethereum/Transaction.h b/libethereum/Transaction.h index 09102e0ba..4271a5f27 100644 --- a/libethereum/Transaction.h +++ b/libethereum/Transaction.h @@ -149,7 +149,7 @@ public: bytes rlp(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return s.out(); } /// @returns the SHA3 hash of the RLP serialisation of this transaction. - h256 sha3(IncludeSignature _sig = WithSignature) const { RLPStream s; streamRLP(s, _sig); return dev::sha3(s.out()); } + h256 sha3(IncludeSignature _sig = WithSignature) const { if (_sig == WithSignature && m_hashWith) return m_hashWith; RLPStream s; streamRLP(s, _sig); auto ret = dev::sha3(s.out()); if (_sig == WithSignature) m_hashWith = ret; return ret; } /// @returns the amount of ETH to be transferred by this (message-call) transaction, in Wei. Synonym for endowment(). u256 value() const { return m_value; } @@ -211,6 +211,7 @@ private: bytes m_data; ///< The data associated with the transaction, or the initialiser if it's a creation transaction. SignatureStruct m_vrs; ///< The signature of the transaction. Encodes the sender. + mutable h256 m_hashWith; ///< Cached hash of transaction with signature. mutable Address m_sender; ///< Cached sender, determined from signature. mutable bigint m_gasRequired = 0; ///< Memoised amount required for the transaction to run. }; diff --git a/libethereum/TransactionQueue.cpp b/libethereum/TransactionQueue.cpp index 57429d32c..1bfdf535a 100644 --- a/libethereum/TransactionQueue.cpp +++ b/libethereum/TransactionQueue.cpp @@ -38,26 +38,56 @@ ImportResult TransactionQueue::import(bytesConstRef _transactionRLP, ImportCallb UpgradableGuard l(m_lock); // TODO: keep old transactions around and check in State for nonce validity - if (m_known.count(h)) + auto ir = check_WITH_LOCK(h, _ik); + if (ir != ImportResult::Success) + return ir; + + Transaction t(_transactionRLP, CheckTransaction::Everything); + UpgradeGuard ul(l); + return manageImport_WITH_LOCK(h, t, _cb); +} + +ImportResult TransactionQueue::check_WITH_LOCK(h256 const& _h, IfDropped _ik) +{ + if (m_known.count(_h)) return ImportResult::AlreadyKnown; - if (m_dropped.count(h) && _ik == IfDropped::Ignore) + if (m_dropped.count(_h) && _ik == IfDropped::Ignore) return ImportResult::AlreadyInChain; + return ImportResult::Success; +} + +ImportResult TransactionQueue::import(Transaction const& _transaction, ImportCallback const& _cb, IfDropped _ik) +{ + // Check if we already know this transaction. + h256 h = _transaction.sha3(WithSignature); + + UpgradableGuard l(m_lock); + // TODO: keep old transactions around and check in State for nonce validity + + auto ir = check_WITH_LOCK(h, _ik); + if (ir != ImportResult::Success) + return ir; + + UpgradeGuard ul(l); + return manageImport_WITH_LOCK(h, _transaction, _cb); +} + +ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb) +{ try { // Check validity of _transactionRLP as a transaction. To do this we just deserialise and attempt to determine the sender. // If it doesn't work, the signature is bad. // The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction). - Transaction t(_transactionRLP, CheckTransaction::Everything); - UpgradeGuard ul(l); // If valid, append to blocks. - insertCurrent_WITH_LOCK(make_pair(h, t)); - m_known.insert(h); + insertCurrent_WITH_LOCK(make_pair(_h, _transaction)); + m_known.insert(_h); if (_cb) - m_callbacks[h] = _cb; - ctxq << "Queued vaguely legit-looking transaction" << h; + m_callbacks[_h] = _cb; + ctxq << "Queued vaguely legit-looking transaction" << _h; m_onReady(); } catch (Exception const& _e) diff --git a/libethereum/TransactionQueue.h b/libethereum/TransactionQueue.h index 16bc34641..69e1c935f 100644 --- a/libethereum/TransactionQueue.h +++ b/libethereum/TransactionQueue.h @@ -49,6 +49,7 @@ class TransactionQueue public: using ImportCallback = std::function; + ImportResult import(Transaction const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore); ImportResult import(bytes const& _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore) { return import(&_tx, _cb, _ik); } ImportResult import(bytesConstRef _tx, ImportCallback const& _cb = ImportCallback(), IfDropped _ik = IfDropped::Ignore); @@ -65,6 +66,9 @@ public: template Handler onReady(T const& _t) { return m_onReady.add(_t); } private: + ImportResult check_WITH_LOCK(h256 const& _h, IfDropped _ik); + ImportResult manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb); + void insertCurrent_WITH_LOCK(std::pair const& _p); bool removeCurrent_WITH_LOCK(h256 const& _txHash);