Browse Source

Fix various aspects of chain-re-import.

cl-refactor
Gav Wood 10 years ago
parent
commit
c5f80a9dde
  1. 101
      libethereum/BlockChain.cpp
  2. 4
      libethereum/BlockChain.h
  3. 22
      libethereum/BlockQueue.cpp
  4. 4
      libethereum/BlockQueue.h
  5. 2
      libethereum/CanonBlockChain.cpp
  6. 4
      libethereum/CanonBlockChain.h
  7. 4
      libethereum/Client.cpp
  8. 35
      libethereum/State.cpp

101
libethereum/BlockChain.cpp

@ -19,10 +19,10 @@
* @date 2014 * @date 2014
*/ */
#include <leveldb/db.h>
#include "BlockChain.h" #include "BlockChain.h"
#include <leveldb/db.h>
#include <boost/timer.hpp>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
#include <test/JsonSpiritHeaders.h> #include <test/JsonSpiritHeaders.h>
#include <libdevcore/Common.h> #include <libdevcore/Common.h>
@ -43,6 +43,7 @@ using namespace dev::eth;
namespace js = json_spirit; namespace js = json_spirit;
#define ETH_CATCH 1 #define ETH_CATCH 1
#define ETH_TIMED_IMPORTS 0
std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc) std::ostream& dev::eth::operator<<(std::ostream& _out, BlockChain const& _bc)
{ {
@ -174,6 +175,8 @@ void BlockChain::close()
m_blocks.clear(); m_blocks.clear();
} }
#define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {}
void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned, unsigned)> const& _progress) void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned, unsigned)> const& _progress)
{ {
unsigned originalNumber = number(); unsigned originalNumber = number();
@ -181,6 +184,7 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
// Keep extras DB around, but under a temp name // Keep extras DB around, but under a temp name
delete m_extrasDB; delete m_extrasDB;
m_extrasDB = nullptr; m_extrasDB = nullptr;
IGNORE_EXCEPTIONS(boost::filesystem::remove_all(_path + "/details.old"));
boost::filesystem::rename(_path + "/details", _path + "/details.old"); boost::filesystem::rename(_path + "/details", _path + "/details.old");
ldb::DB* oldExtrasDB; ldb::DB* oldExtrasDB;
ldb::Options o; ldb::Options o;
@ -189,7 +193,7 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
ldb::DB::Open(o, _path + "/details", &m_extrasDB); ldb::DB::Open(o, _path + "/details", &m_extrasDB);
// Open a fresh state DB // Open a fresh state DB
OverlayDB db = State::openDB(_path, WithExisting::Kill); State s(Address(), State::openDB(_path, WithExisting::Kill), BaseState::CanonGenesis);
// Clear all memos ready for replay. // Clear all memos ready for replay.
m_details.clear(); m_details.clear();
@ -201,11 +205,21 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
m_lastLastHashes.clear(); m_lastLastHashes.clear();
m_lastBlockHash = genesisHash(); m_lastBlockHash = genesisHash();
for (unsigned d = 0; d < originalNumber; ++d) h256 lastHash = genesisHash();
for (unsigned d = 1; d < originalNumber; ++d)
{ {
try try
{ {
import(block(queryExtras<BlockHash, ExtraBlockHash>(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value), db); bytes b = block(queryExtras<BlockHash, ExtraBlockHash>(h256(u256(d)), m_blockHashes, x_blockHashes, NullBlockHash, oldExtrasDB).value);
BlockInfo bi(b);
if (bi.parentHash != lastHash)
{
cwarn << "DISJOINT CHAIN DETECTED; " << bi.hash.abridged() << "#" << d << " -> parent is" << bi.parentHash.abridged() << "; expected" << lastHash.abridged() << "#" << (d - 1);
return;
}
lastHash = bi.hash;
import(b, s.db(), true);
} }
catch (...) catch (...)
{ {
@ -258,7 +272,7 @@ h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max
_bq.tick(*this); _bq.tick(*this);
vector<bytes> blocks; vector<bytes> blocks;
_bq.drain(blocks); _bq.drain(blocks, _max);
h256s ret; h256s ret;
for (auto const& block: blocks) for (auto const& block: blocks)
@ -266,9 +280,6 @@ h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max
try try
{ {
for (auto h: import(block, _stateDB)) for (auto h: import(block, _stateDB))
if (!_max--)
break;
else
ret.push_back(h); ret.push_back(h);
} }
catch (UnknownParent) catch (UnknownParent)
@ -288,11 +299,11 @@ h256s BlockChain::sync(BlockQueue& _bq, OverlayDB const& _stateDB, unsigned _max
return ret; return ret;
} }
h256s BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB) noexcept h256s BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force) noexcept
{ {
try try
{ {
return import(_block, _stateDB); return import(_block, _stateDB, _force);
} }
catch (...) catch (...)
{ {
@ -301,8 +312,18 @@ h256s BlockChain::attemptImport(bytes const& _block, OverlayDB const& _stateDB)
} }
} }
h256s BlockChain::import(bytes const& _block, OverlayDB const& _db) h256s BlockChain::import(bytes const& _block, OverlayDB const& _db, bool _force)
{ {
#if ETH_TIMED_IMPORTS
boost::timer total;
double preliminaryChecks;
double enactment;
double collation;
double writing;
double checkBest;
boost::timer t;
#endif
// VERIFY: populates from the block and checks the block is internally coherent. // VERIFY: populates from the block and checks the block is internally coherent.
BlockInfo bi; BlockInfo bi;
@ -329,7 +350,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
auto newHash = BlockInfo::headerHash(_block); auto newHash = BlockInfo::headerHash(_block);
// Check block doesn't already exist first! // Check block doesn't already exist first!
if (isKnown(newHash)) if (isKnown(newHash) && !_force)
{ {
clog(BlockChainNote) << newHash << ": Not new."; clog(BlockChainNote) << newHash << ": Not new.";
BOOST_THROW_EXCEPTION(AlreadyHaveBlock()); BOOST_THROW_EXCEPTION(AlreadyHaveBlock());
@ -362,6 +383,11 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
clog(BlockChainNote) << "Attempting import of " << newHash.abridged() << "..."; clog(BlockChainNote) << "Attempting import of " << newHash.abridged() << "...";
#if ETH_TIMED_IMPORTS
preliminaryChecks = t.elapsed();
t.restart();
#endif
u256 td; u256 td;
#if ETH_CATCH #if ETH_CATCH
try try
@ -371,6 +397,7 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
// Get total difficulty increase and update state, checking it. // Get total difficulty increase and update state, checking it.
State s(bi.coinbaseAddress, _db); State s(bi.coinbaseAddress, _db);
auto tdIncrease = s.enactOn(&_block, bi, *this); auto tdIncrease = s.enactOn(&_block, bi, *this);
BlockLogBlooms blb; BlockLogBlooms blb;
BlockReceipts br; BlockReceipts br;
for (unsigned i = 0; i < s.pending().size(); ++i) for (unsigned i = 0; i < s.pending().size(); ++i)
@ -381,6 +408,11 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
s.cleanup(true); s.cleanup(true);
td = pd.totalDifficulty + tdIncrease; td = pd.totalDifficulty + tdIncrease;
#if ETH_TIMED_IMPORTS
enactment = t.elapsed();
t.restart();
#endif
#if ETH_PARANOIA #if ETH_PARANOIA
checkConsistency(); checkConsistency();
#endif #endif
@ -397,10 +429,6 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
m_details[newHash] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {}); m_details[newHash] = BlockDetails((unsigned)pd.number + 1, td, bi.parentHash, {});
m_details[bi.parentHash].children.push_back(newHash); m_details[bi.parentHash].children.push_back(newHash);
} }
{
WriteGuard l(x_blockHashes);
m_blockHashes[h256(bi.number)].value = newHash;
}
h256s alteredBlooms; h256s alteredBlooms;
{ {
WriteGuard l(x_blocksBlooms); WriteGuard l(x_blocksBlooms);
@ -415,6 +443,16 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
m_blocksBlooms[alteredBlooms.back()].blooms[o] |= blockBloom; m_blocksBlooms[alteredBlooms.back()].blooms[o] |= blockBloom;
} }
} }
{
WriteGuard l(x_logBlooms);
m_logBlooms[newHash] = blb;
}
{
WriteGuard l(x_receipts);
m_receipts[newHash] = br;
}
// TODO: FIX: URGENT!!!!! only put these into database when the block is on the main chain.
// Collate transaction hashes and remember who they were. // Collate transaction hashes and remember who they were.
h256s newTransactionAddresses; h256s newTransactionAddresses;
{ {
@ -429,14 +467,15 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
} }
} }
{ {
WriteGuard l(x_logBlooms); WriteGuard l(x_blockHashes);
m_logBlooms[newHash] = blb; m_blockHashes[h256(bi.number)].value = newHash;
}
{
WriteGuard l(x_receipts);
m_receipts[newHash] = br;
} }
#if ETH_TIMED_IMPORTS
collation = t.elapsed();
t.restart();
#endif
{ {
ReadGuard l1(x_blocksBlooms); ReadGuard l1(x_blocksBlooms);
ReadGuard l2(x_details); ReadGuard l2(x_details);
@ -456,6 +495,11 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[h].rlp())); m_extrasDB->Put(m_writeOptions, toSlice(h, ExtraBlocksBlooms), (ldb::Slice)dev::ref(m_blocksBlooms[h].rlp()));
} }
#if ETH_TIMED_IMPORTS
writing = t.elapsed();
t.restart();
#endif
#if ETH_PARANOIA #if ETH_PARANOIA
checkConsistency(); checkConsistency();
#endif #endif
@ -514,6 +558,17 @@ h256s BlockChain::import(bytes const& _block, OverlayDB const& _db)
{ {
clog(BlockChainNote) << " Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << ")"; clog(BlockChainNote) << " Imported but not best (oTD:" << details(last).totalDifficulty << " > TD:" << td << ")";
} }
#if ETH_TIMED_IMPORTS
checkBest = t.elapsed();
cnote << "Import took:" << total.elapsed();
cnote << "preliminaryChecks:" << preliminaryChecks;
cnote << "enactment:" << enactment;
cnote << "collation:" << collation;
cnote << "writing:" << writing;
cnote << "checkBest:" << checkBest;
#endif
return ret; return ret;
} }

4
libethereum/BlockChain.h

@ -103,11 +103,11 @@ public:
/// Attempt to import the given block directly into the CanonBlockChain and sync with the state DB. /// Attempt to import the given block directly into the CanonBlockChain and sync with the state DB.
/// @returns the block hashes of any blocks that came into/went out of the canonical block chain. /// @returns the block hashes of any blocks that came into/went out of the canonical block chain.
h256s attemptImport(bytes const& _block, OverlayDB const& _stateDB) noexcept; h256s attemptImport(bytes const& _block, OverlayDB const& _stateDB, bool _force = false) noexcept;
/// Import block into disk-backed DB /// Import block into disk-backed DB
/// @returns the block hashes of any blocks that came into/went out of the canonical block chain. /// @returns the block hashes of any blocks that came into/went out of the canonical block chain.
h256s import(bytes const& _block, OverlayDB const& _stateDB); h256s import(bytes const& _block, OverlayDB const& _stateDB, bool _force = false);
/// Returns true if the given block is known (though not necessarily a part of the canon chain). /// Returns true if the given block is known (though not necessarily a part of the canon chain).
bool isKnown(h256 const& _hash) const; bool isKnown(h256 const& _hash) const;

22
libethereum/BlockQueue.cpp

@ -114,13 +114,29 @@ void BlockQueue::tick(BlockChain const& _bc)
m_future.erase(m_future.begin(), m_future.upper_bound(t)); m_future.erase(m_future.begin(), m_future.upper_bound(t));
} }
void BlockQueue::drain(std::vector<bytes>& o_out) template <class T> T advanced(T _t, unsigned _n)
{
std::advance(_t, _n);
return _t;
}
void BlockQueue::drain(std::vector<bytes>& o_out, unsigned _max)
{ {
WriteGuard l(m_lock); WriteGuard l(m_lock);
if (m_drainingSet.empty()) if (m_drainingSet.empty())
{ {
swap(o_out, m_ready); o_out.resize(min<unsigned>(_max, m_ready.size()));
swap(m_drainingSet, m_readySet); for (unsigned i = 0; i < o_out.size(); ++i)
swap(o_out[i], m_ready[i]);
m_ready.erase(m_ready.begin(), advanced(m_ready.begin(), o_out.size()));
for (auto const& bs: o_out)
{
auto h = sha3(bs);
m_drainingSet.insert(h);
m_readySet.erase(h);
}
// swap(o_out, m_ready);
// swap(m_drainingSet, m_readySet);
} }
} }

4
libethereum/BlockQueue.h

@ -61,9 +61,9 @@ public:
/// Notes that time has moved on and some blocks that used to be "in the future" may no be valid. /// Notes that time has moved on and some blocks that used to be "in the future" may no be valid.
void tick(BlockChain const& _bc); void tick(BlockChain const& _bc);
/// Grabs the blocks that are ready, giving them in the correct order for insertion into the chain. /// Grabs at most @a _max of the blocks that are ready, giving them in the correct order for insertion into the chain.
/// Don't forget to call doneDrain() once you're done importing. /// Don't forget to call doneDrain() once you're done importing.
void drain(std::vector<bytes>& o_out); void drain(std::vector<bytes>& o_out, unsigned _max);
/// Must be called after a drain() call. Notes that the drained blocks have been imported into the blockchain, so we can forget about them. /// Must be called after a drain() call. Notes that the drained blocks have been imported into the blockchain, so we can forget about them.
void doneDrain() { WriteGuard l(m_lock); m_drainingSet.clear(); } void doneDrain() { WriteGuard l(m_lock); m_drainingSet.clear(); }

2
libethereum/CanonBlockChain.cpp

@ -92,6 +92,6 @@ bytes CanonBlockChain::createGenesisBlock()
return block.out(); return block.out();
} }
CanonBlockChain::CanonBlockChain(std::string const& _path, WithExisting _we): BlockChain(CanonBlockChain::createGenesisBlock(), _path, _we) CanonBlockChain::CanonBlockChain(std::string const& _path, WithExisting _we, ProgressCallback const& _pc): BlockChain(CanonBlockChain::createGenesisBlock(), _path, _we, _pc)
{ {
} }

4
libethereum/CanonBlockChain.h

@ -55,8 +55,8 @@ std::map<Address, Account> const& genesisState();
class CanonBlockChain: public BlockChain class CanonBlockChain: public BlockChain
{ {
public: public:
CanonBlockChain(WithExisting _we = WithExisting::Trust): CanonBlockChain(std::string(), _we) {} CanonBlockChain(WithExisting _we = WithExisting::Trust, ProgressCallback const& _pc = ProgressCallback()): CanonBlockChain(std::string(), _we, _pc) {}
CanonBlockChain(std::string const& _path, WithExisting _we = WithExisting::Trust); CanonBlockChain(std::string const& _path, WithExisting _we = WithExisting::Trust, ProgressCallback const& _pc = ProgressCallback());
~CanonBlockChain() {} ~CanonBlockChain() {}
/// @returns the genesis block header. /// @returns the genesis block header.

4
libethereum/Client.cpp

@ -120,7 +120,7 @@ void BasicGasPricer::update(BlockChain const& _bc)
Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId, int _miners): Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId, int _miners):
Worker("eth"), Worker("eth"),
m_vc(_dbPath), m_vc(_dbPath),
m_bc(_dbPath, max(m_vc.action(), _forceAction)), m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "..." << endl; }),
m_gp(new TrivialGasPricer), m_gp(new TrivialGasPricer),
m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))),
m_preMine(Address(), m_stateDB), m_preMine(Address(), m_stateDB),
@ -145,7 +145,7 @@ Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _for
Client::Client(p2p::Host* _extNet, std::shared_ptr<GasPricer> _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId, int _miners): Client::Client(p2p::Host* _extNet, std::shared_ptr<GasPricer> _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId, int _miners):
Worker("eth"), Worker("eth"),
m_vc(_dbPath), m_vc(_dbPath),
m_bc(_dbPath, max(m_vc.action(), _forceAction)), m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "..." << endl; }),
m_gp(_gp), m_gp(_gp),
m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))), m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))),
m_preMine(Address(), m_stateDB), m_preMine(Address(), m_stateDB),

35
libethereum/State.cpp

@ -43,6 +43,7 @@ using namespace dev;
using namespace dev::eth; using namespace dev::eth;
#define ctrace clog(StateTrace) #define ctrace clog(StateTrace)
#define ETH_TIMED_ENACTMENTS 0
static const u256 c_blockReward = 1500 * finney; static const u256 c_blockReward = 1500 * finney;
@ -353,16 +354,48 @@ bool State::sync(BlockChain const& _bc, h256 _block, BlockInfo const& _bi)
u256 State::enactOn(bytesConstRef _block, BlockInfo const& _bi, BlockChain const& _bc) u256 State::enactOn(bytesConstRef _block, BlockInfo const& _bi, BlockChain const& _bc)
{ {
#if ETH_TIMED_ENACTMENTS
boost::timer t;
double populateVerify;
double populateGrand;
double syncReset;
double enactment;
#endif
// Check family: // Check family:
BlockInfo biParent(_bc.block(_bi.parentHash)); BlockInfo biParent(_bc.block(_bi.parentHash));
_bi.verifyParent(biParent); _bi.verifyParent(biParent);
#if ETH_TIMED_ENACTMENTS
populateVerify = t.elapsed();
t.restart();
#endif
BlockInfo biGrandParent; BlockInfo biGrandParent;
if (biParent.number) if (biParent.number)
biGrandParent.populate(_bc.block(biParent.parentHash)); biGrandParent.populate(_bc.block(biParent.parentHash));
#if ETH_TIMED_ENACTMENTS
populateGrand = t.elapsed();
t.restart();
#endif
sync(_bc, _bi.parentHash); sync(_bc, _bi.parentHash);
resetCurrent(); resetCurrent();
#if ETH_TIMED_ENACTMENTS
syncReset = t.elapsed();
t.restart();
#endif
m_previousBlock = biParent; m_previousBlock = biParent;
return enact(_block, _bc); auto ret = enact(_block, _bc);
#if ETH_TIMED_ENACTMENTS
enactment = t.elapsed();
cnote << "popVer/popGrand/syncReset/enactment = " << populateVerify << "/" << populateGrand << "/" << syncReset << "/" << enactment;
#endif
return ret;
} }
map<Address, u256> State::addresses() const map<Address, u256> State::addresses() const

Loading…
Cancel
Save