Don't drop DB when chaning DB format.

Don't die when geth is run.
Repotting.
Fixed #2354.

libdevcore/Common.h

@@ -64,6 +64,8 @@ using byte = uint8_t;
 
 #define DEV_IGNORE_EXCEPTIONS(X) try { X; } catch (...) {}
 
+#define DEV_IF_NO_ELSE(X) if(!(X)){}else
+
 namespace dev
 {
 

libdevcore/CommonData.h

@@ -50,8 +50,8 @@ enum class HexPrefix
 /// Convert a series of bytes to the corresponding string of hex duplets.
 /// @param _w specifies the width of the first of the elements. Defaults to two - enough to represent a byte.
 /// @example toHex("A\x69") == "4169"
-template <class _T>
-std::string toHex(_T const& _data, int _w = 2, HexPrefix _prefix = HexPrefix::DontAdd)
+template <class T>
+std::string toHex(T const& _data, int _w = 2, HexPrefix _prefix = HexPrefix::DontAdd)
 {
 	std::ostringstream ret;
 	unsigned ii = 0;
@@ -99,27 +99,27 @@ bytes asNibbles(bytesConstRef const& _s);
 /// Converts a templated integer value to the big-endian byte-stream represented on a templated collection.
 /// The size of the collection object will be unchanged. If it is too small, it will not represent the
 /// value properly, if too big then the additional elements will be zeroed out.
-/// @a _Out will typically be either std::string or bytes.
-/// @a _T will typically by unsigned, u160, u256 or bigint.
-template <class _T, class _Out>
-inline void toBigEndian(_T _val, _Out& o_out)
+/// @a Out will typically be either std::string or bytes.
+/// @a T will typically by unsigned, u160, u256 or bigint.
+template <class T, class Out>
+inline void toBigEndian(T _val, Out& o_out)
 {
 	for (auto i = o_out.size(); i != 0; _val >>= 8, i--)
 	{
-		_T v = _val & (_T)0xff;
-		o_out[i - 1] = (typename _Out::value_type)(uint8_t)v;
+		T v = _val & (T)0xff;
+		o_out[i - 1] = (typename Out::value_type)(uint8_t)v;
 	}
 }
 
 /// Converts a big-endian byte-stream represented on a templated collection to a templated integer value.
 /// @a _In will typically be either std::string or bytes.
-/// @a _T will typically by unsigned, u160, u256 or bigint.
-template <class _T, class _In>
-inline _T fromBigEndian(_In const& _bytes)
+/// @a T will typically by unsigned, u160, u256 or bigint.
+template <class T, class _In>
+inline T fromBigEndian(_In const& _bytes)
 {
-	_T ret = (_T)0;
+	T ret = (T)0;
 	for (auto i: _bytes)
-		ret = (_T)((ret << 8) | (byte)(typename std::make_unsigned<typename _In::value_type>::type)i);
+		ret = (T)((ret << 8) | (byte)(typename std::make_unsigned<typename _In::value_type>::type)i);
 	return ret;
 }
 
@@ -131,11 +131,11 @@ inline bytes toBigEndian(u160 _val) { bytes ret(20); toBigEndian(_val, ret); ret
 
 /// Convenience function for toBigEndian.
 /// @returns a byte array just big enough to represent @a _val.
-template <class _T>
-inline bytes toCompactBigEndian(_T _val, unsigned _min = 0)
+template <class T>
+inline bytes toCompactBigEndian(T _val, unsigned _min = 0)
 {
 	int i = 0;
-	for (_T v = _val; v; ++i, v >>= 8) {}
+	for (T v = _val; v; ++i, v >>= 8) {}
 	bytes ret(std::max<unsigned>(_min, i), 0);
 	toBigEndian(_val, ret);
 	return ret;
@@ -147,11 +147,11 @@ inline bytes toCompactBigEndian(byte _val, unsigned _min = 0)
 
 /// Convenience function for toBigEndian.
 /// @returns a string just big enough to represent @a _val.
-template <class _T>
-inline std::string toCompactBigEndianString(_T _val, unsigned _min = 0)
+template <class T>
+inline std::string toCompactBigEndianString(T _val, unsigned _min = 0)
 {
 	int i = 0;
-	for (_T v = _val; v; ++i, v >>= 8) {}
+	for (T v = _val; v; ++i, v >>= 8) {}
 	std::string ret(std::max<unsigned>(_min, i), '\0');
 	toBigEndian(_val, ret);
 	return ret;
@@ -179,8 +179,8 @@ std::string escaped(std::string const& _s, bool _all = true);
 /// Determines the length of the common prefix of the two collections given.
 /// @returns the number of elements both @a _t and @a _u share, in order, at the beginning.
 /// @example commonPrefix("Hello world!", "Hello, world!") == 5
-template <class _T, class _U>
-unsigned commonPrefix(_T const& _t, _U const& _u)
+template <class T, class _U>
+unsigned commonPrefix(T const& _t, _U const& _u)
 {
 	unsigned s = std::min<unsigned>(_t.size(), _u.size());
 	for (unsigned i = 0;; ++i)
@@ -196,8 +196,8 @@ std::string randomWord();
 // General datatype convenience functions.
 
 /// Determine bytes required to encode the given integer value. @returns 0 if @a _i is zero.
-template <class _T>
-inline unsigned bytesRequired(_T _i)
+template <class T>
+inline unsigned bytesRequired(T _i)
 {
 	unsigned i = 0;
 	for (; _i != 0; ++i, _i >>= 8) {}
@@ -206,39 +206,39 @@ inline unsigned bytesRequired(_T _i)
 
 /// Trims a given number of elements from the front of a collection.
 /// Only works for POD element types.
-template <class _T>
-void trimFront(_T& _t, unsigned _elements)
+template <class T>
+void trimFront(T& _t, unsigned _elements)
 {
-	static_assert(std::is_pod<typename _T::value_type>::value, "");
+	static_assert(std::is_pod<typename T::value_type>::value, "");
 	memmove(_t.data(), _t.data() + _elements, (_t.size() - _elements) * sizeof(_t[0]));
 	_t.resize(_t.size() - _elements);
 }
 
 /// Pushes an element on to the front of a collection.
 /// Only works for POD element types.
-template <class _T, class _U>
-void pushFront(_T& _t, _U _e)
+template <class T, class _U>
+void pushFront(T& _t, _U _e)
 {
-	static_assert(std::is_pod<typename _T::value_type>::value, "");
+	static_assert(std::is_pod<typename T::value_type>::value, "");
 	_t.push_back(_e);
 	memmove(_t.data() + 1, _t.data(), (_t.size() - 1) * sizeof(_e));
 	_t[0] = _e;
 }
 
 /// Concatenate two vectors of elements of POD types.
-template <class _T>
-inline std::vector<_T>& operator+=(std::vector<typename std::enable_if<std::is_pod<_T>::value, _T>::type>& _a, std::vector<_T> const& _b)
+template <class T>
+inline std::vector<T>& operator+=(std::vector<typename std::enable_if<std::is_pod<T>::value, T>::type>& _a, std::vector<T> const& _b)
 {
 	auto s = _a.size();
 	_a.resize(_a.size() + _b.size());
-	memcpy(_a.data() + s, _b.data(), _b.size() * sizeof(_T));
+	memcpy(_a.data() + s, _b.data(), _b.size() * sizeof(T));
 	return _a;
 
 }
 
 /// Concatenate two vectors of elements.
-template <class _T>
-inline std::vector<_T>& operator+=(std::vector<typename std::enable_if<!std::is_pod<_T>::value, _T>::type>& _a, std::vector<_T> const& _b)
+template <class T>
+inline std::vector<T>& operator+=(std::vector<typename std::enable_if<!std::is_pod<T>::value, T>::type>& _a, std::vector<T> const& _b)
 {
 	_a.reserve(_a.size() + _b.size());
 	for (auto& i: _b)
@@ -289,16 +289,16 @@ template <class T, class U> std::vector<T> operator+(std::vector<T> _a, U const&
 }
 
 /// Concatenate two vectors of elements.
-template <class _T>
-inline std::vector<_T> operator+(std::vector<_T> const& _a, std::vector<_T> const& _b)
+template <class T>
+inline std::vector<T> operator+(std::vector<T> const& _a, std::vector<T> const& _b)
 {
-	std::vector<_T> ret(_a);
+	std::vector<T> ret(_a);
 	return ret += _b;
 }
 
 /// Merge two sets of elements.
-template <class _T>
-inline std::set<_T>& operator+=(std::set<_T>& _a, std::set<_T> const& _b)
+template <class T>
+inline std::set<T>& operator+=(std::set<T>& _a, std::set<T> const& _b)
 {
 	for (auto& i: _b)
 		_a.insert(i);
@@ -306,13 +306,28 @@ inline std::set<_T>& operator+=(std::set<_T>& _a, std::set<_T> const& _b)
 }
 
 /// Merge two sets of elements.
-template <class _T>
-inline std::set<_T> operator+(std::set<_T> const& _a, std::set<_T> const& _b)
+template <class T>
+inline std::set<T> operator+(std::set<T> const& _a, std::set<T> const& _b)
 {
-	std::set<_T> ret(_a);
+	std::set<T> ret(_a);
 	return ret += _b;
 }
 
+template <class A, class B>
+std::unordered_map<A, B>& operator+=(std::unordered_map<A, B>& _x, std::unordered_map<A, B> const& _y)
+{
+	for (auto const& i: _y)
+		_x.insert(i);
+	return _x;
+}
+
+template <class A, class B>
+std::unordered_map<A, B> operator+(std::unordered_map<A, B> const& _x, std::unordered_map<A, B> const& _y)
+{
+	std::unordered_map<A, B> ret(_x);
+	return ret += _y;
+}
+
 /// Make normal string from fixed-length string.
 std::string toString(string32 const& _s);
 

libethereum/BasicGasPricer.cpp

@@ -0,0 +1,95 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file BasicGasPricer.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2015
+ */
+
+#include <boost/math/distributions/normal.hpp>
+#include "BasicGasPricer.h"
+#include "BlockChain.h"
+using namespace std;
+using namespace dev;
+using namespace dev::eth;
+
+void BasicGasPricer::update(BlockChain const& _bc)
+{
+	unsigned c = 0;
+	h256 p = _bc.currentHash();
+	m_gasPerBlock = _bc.info(p).gasLimit;
+
+	map<u256, u256> dist;
+	u256 total = 0;
+
+	// make gasPrice versus gasUsed distribution for the last 1000 blocks
+	while (c < 1000 && p)
+	{
+		BlockInfo bi = _bc.info(p);
+		if (bi.transactionsRoot != EmptyTrie)
+		{
+			auto bb = _bc.block(p);
+			RLP r(bb);
+			BlockReceipts brs(_bc.receipts(bi.hash()));
+			size_t i = 0;
+			for (auto const& tr: r[1])
+			{
+				Transaction tx(tr.data(), CheckTransaction::None);
+				u256 gu = brs.receipts[i].gasUsed();
+				dist[tx.gasPrice()] += gu;
+				total += gu;
+				i++;
+			}
+		}
+		p = bi.parentHash;
+		++c;
+	}
+
+	// fill m_octiles with weighted gasPrices
+	if (total > 0)
+	{
+		m_octiles[0] = dist.begin()->first;
+
+		// calc mean
+		u256 mean = 0;
+		for (auto const& i: dist)
+			mean += i.first * i.second;
+		mean /= total;
+
+		// calc standard deviation
+		u256 sdSquared = 0;
+		for (auto const& i: dist)
+			sdSquared += i.second * (i.first - mean) * (i.first - mean);
+		sdSquared /= total;
+
+		if (sdSquared)
+		{
+			long double sd = sqrt(sdSquared.convert_to<long double>());
+			long double normalizedSd = sd / mean.convert_to<long double>();
+
+			// calc octiles normalized to gaussian distribution
+			boost::math::normal gauss(1.0, (normalizedSd > 0.01) ? normalizedSd : 0.01);
+			for (size_t i = 1; i < 8; i++)
+				m_octiles[i] = u256(mean.convert_to<long double>() * boost::math::quantile(gauss, i / 8.0));
+			m_octiles[8] = dist.rbegin()->first;
+		}
+		else
+		{
+			for (size_t i = 0; i < 9; i++)
+				m_octiles[i] = (i + 1) * mean / 5;
+		}
+	}
+}

libethereum/BasicGasPricer.h

@@ -0,0 +1,53 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file BasicGasPricer.h
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#pragma once
+
+#include <array>
+#include "GasPricer.h"
+
+namespace dev
+{
+namespace eth
+{
+
+class BasicGasPricer: public GasPricer
+{
+public:
+	explicit BasicGasPricer(u256 _weiPerRef, u256 _refsPerBlock): m_weiPerRef(_weiPerRef), m_refsPerBlock(_refsPerBlock) {}
+
+	void setRefPrice(u256 _weiPerRef) { if ((bigint)m_refsPerBlock * _weiPerRef > std::numeric_limits<u256>::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_weiPerRef = _weiPerRef; }
+	void setRefBlockFees(u256 _refsPerBlock) { if ((bigint)m_weiPerRef * _refsPerBlock > std::numeric_limits<u256>::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_refsPerBlock = _refsPerBlock; }
+
+	u256 ask(State const&) const override { return m_weiPerRef * m_refsPerBlock / m_gasPerBlock; }
+	u256 bid(TransactionPriority _p = TransactionPriority::Medium) const override { return m_octiles[(int)_p] > 0 ? m_octiles[(int)_p] : (m_weiPerRef * m_refsPerBlock / m_gasPerBlock); }
+
+	void update(BlockChain const& _bc) override;
+
+private:
+	u256 m_weiPerRef;
+	u256 m_refsPerBlock;
+	u256 m_gasPerBlock = 3141592;
+	std::array<u256, 9> m_octiles;
+};
+
+}
+}

libethereum/BlockChain.cpp

@@ -127,7 +127,7 @@ static const unsigned c_minCacheSize = 1024 * 1024 * 32;
 
 #endif
 
-BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, WithExisting _we, ProgressCallback const& _p)
+BlockChain::BlockChain(bytes const& _genesisBlock, std::string const& _path, WithExisting _we, ProgressCallback const& _p)
 {
 	// initialise deathrow.
 	m_cacheUsage.resize(c_collectionQueueSize);
@@ -137,8 +137,7 @@ BlockChain::BlockChain(bytes const& _genesisBlock, std::string _path, WithExisti
 	m_genesisBlock = _genesisBlock;
 	m_genesisHash = sha3(RLP(m_genesisBlock)[0].data());
 
-	open(_path, _we);
-	if (_we == WithExisting::Verify)
+	if (open(_path, _we) != c_minorProtocolVersion)
 		rebuild(_path, _p);
 }
 
@@ -147,24 +146,41 @@ BlockChain::~BlockChain()
 	close();
 }
 
-void BlockChain::open(std::string const& _path, WithExisting _we)
+unsigned BlockChain::open(std::string const& _path, WithExisting _we)
 {
-	std::string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
-	boost::filesystem::create_directories(path);
+	string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
+	string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4));
+	string extrasPath = chainPath + "/" + toString(c_databaseVersion);
+
+	boost::filesystem::create_directories(extrasPath);
+
+	bytes status = contents(extrasPath + "/minor");
+	unsigned lastMinor = c_minorProtocolVersion;
+	DEV_IGNORE_EXCEPTIONS(lastMinor = (unsigned)RLP(status));
+	if (c_minorProtocolVersion != lastMinor)
+	{
+		cnote << "Killing extras database (DB minor version:" << lastMinor << " != our miner version: " << c_minorProtocolVersion << ").";
+		DEV_IGNORE_EXCEPTIONS(boost::filesystem::remove_all(extrasPath + "/details.old"));
+		boost::filesystem::rename(extrasPath + "/extras", extrasPath + "/extras.old");
+		boost::filesystem::remove_all(extrasPath + "/state");
+		writeFile(extrasPath + "/minor", rlp(c_minorProtocolVersion));
+		lastMinor = (unsigned)RLP(status);
+	}
 	if (_we == WithExisting::Kill)
 	{
-		boost::filesystem::remove_all(path + "/blocks");
-		boost::filesystem::remove_all(path + "/details");
+		cnote << "Killing blockchain & extras database (WithExisting::Kill).";
+		boost::filesystem::remove_all(chainPath + "/blocks");
+		boost::filesystem::remove_all(extrasPath + "/extras");
 	}
 
 	ldb::Options o;
 	o.create_if_missing = true;
 	o.max_open_files = 256;
-	ldb::DB::Open(o, path + "/blocks", &m_blocksDB);
-	ldb::DB::Open(o, path + "/details", &m_extrasDB);
+	ldb::DB::Open(o, chainPath + "/blocks", &m_blocksDB);
+	ldb::DB::Open(o, extrasPath + "/extras", &m_extrasDB);
 	if (!m_blocksDB || !m_extrasDB)
 	{
-		if (boost::filesystem::space(path + "/blocks").available < 1024)
+		if (boost::filesystem::space(chainPath + "/blocks").available < 1024)
 		{
 			cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing.";
 			BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace());
@@ -194,7 +210,8 @@ void BlockChain::open(std::string const& _path, WithExisting _we)
 	m_lastBlockHash = l.empty() ? m_genesisHash : *(h256*)l.data();
 	m_lastBlockNumber = number(m_lastBlockHash);
 
-	cnote << "Opened blockchain DB. Latest: " << currentHash();
+	cnote << "Opened blockchain DB. Latest: " << currentHash() << (lastMinor == c_minorProtocolVersion ? "(rebuild not needed)" : "*** REBUILD NEEDED ***");
+	return lastMinor;
 }
 
 void BlockChain::close()
@@ -208,11 +225,11 @@ void BlockChain::close()
 	m_blocks.clear();
 }
 
-#define IGNORE_EXCEPTIONS(X) try { X; } catch (...) {}
-
 void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned, unsigned)> const& _progress, bool _prepPoW)
 {
-	std::string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
+	string path = _path.empty() ? Defaults::get()->m_dbPath : _path;
+	string chainPath = path + "/" + toHex(m_genesisHash.ref().cropped(0, 4));
+	string extrasPath = chainPath + "/" + toString(c_databaseVersion);
 
 #if ETH_PROFILING_GPERF
 	ProfilerStart("BlockChain_rebuild.log");
@@ -220,16 +237,21 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
 
 	unsigned originalNumber = m_lastBlockNumber;
 
+	///////////////////////////////
+	// TODO
+	// - KILL ALL STATE/CHAIN
+	// - REINSERT ALL BLOCKS
+	///////////////////////////////
+
 	// Keep extras DB around, but under a temp name
 	delete m_extrasDB;
 	m_extrasDB = nullptr;
-	IGNORE_EXCEPTIONS(boost::filesystem::remove_all(path + "/details.old"));
-	boost::filesystem::rename(path + "/details", path + "/details.old");
+	boost::filesystem::rename(path + "/details", path + "/extras.old");
 	ldb::DB* oldExtrasDB;
 	ldb::Options o;
 	o.create_if_missing = true;
-	ldb::DB::Open(o, path + "/details.old", &oldExtrasDB);
-	ldb::DB::Open(o, path + "/details", &m_extrasDB);
+	ldb::DB::Open(o, extrasPath + "/extras.old", &oldExtrasDB);
+	ldb::DB::Open(o, extrasPath + "/extras", &m_extrasDB);
 
 	// Open a fresh state DB
 	State s(State::openDB(path, WithExisting::Kill), BaseState::CanonGenesis);
@@ -289,7 +311,7 @@ void BlockChain::rebuild(std::string const& _path, std::function<void(unsigned,
 #endif
 
 	delete oldExtrasDB;
-	boost::filesystem::remove_all(path + "/details.old");
+	boost::filesystem::remove_all(path + "/extras.old");
 }
 
 LastHashes BlockChain::lastHashes(unsigned _n) const

libethereum/BlockChain.h

@@ -94,7 +94,7 @@ using ProgressCallback = std::function<void(unsigned, unsigned)>;
 class BlockChain
 {
 public:
-	BlockChain(bytes const& _genesisBlock, std::string _path, WithExisting _we, ProgressCallback const& _p = ProgressCallback());
+	BlockChain(bytes const& _genesisBlock, std::string const& _path, WithExisting _we = WithExisting::Trust, ProgressCallback const& _p = ProgressCallback());
 	~BlockChain();
 
 	/// Attempt a database re-open.
@@ -261,7 +261,7 @@ public:
 private:
 	static h256 chunkId(unsigned _level, unsigned _index) { return h256(_index * 0xff + _level); }
 
-	void open(std::string const& _path, WithExisting _we = WithExisting::Trust);
+	unsigned open(std::string const& _path, WithExisting _we = WithExisting::Trust);
 	void close();
 
 	template<class T, unsigned N> T queryExtras(h256 const& _h, std::unordered_map<h256, T>& _m, boost::shared_mutex& _x, T const& _n, ldb::DB* _extrasDB = nullptr) const

libethereum/Client.cpp

@@ -24,7 +24,6 @@
 #include <chrono>
 #include <thread>
 #include <boost/filesystem.hpp>
-#include <boost/math/distributions/normal.hpp>
 #if ETH_JSONRPC || !ETH_TRUE
 #include <jsonrpccpp/client.h>
 #include <jsonrpccpp/client/connectors/httpclient.h>
@@ -38,53 +37,82 @@
 #include "Defaults.h"
 #include "Executive.h"
 #include "EthereumHost.h"
+#include "Utility.h"
 using namespace std;
 using namespace dev;
 using namespace dev::eth;
 using namespace p2p;
 
-VersionChecker::VersionChecker(string const& _dbPath):
-	m_path(_dbPath.size() ? _dbPath : Defaults::dbPath())
+std::ostream& dev::eth::operator<<(std::ostream& _out, ActivityReport const& _r)
 {
-	bytes statusBytes = contents(m_path + "/status");
-	RLP status(statusBytes);
-	try
-	{
-		auto protocolVersion = (unsigned)status[0];
-		(void)protocolVersion;
-		auto minorProtocolVersion = (unsigned)status[1];
-		auto databaseVersion = (unsigned)status[2];
-		h256 ourGenesisHash = CanonBlockChain::genesis().hash();
-		auto genesisHash = status.itemCount() > 3 ? (h256)status[3] : ourGenesisHash;
-
-		m_action =
-			databaseVersion != c_databaseVersion || genesisHash != ourGenesisHash ?
-				WithExisting::Kill
-			: minorProtocolVersion != eth::c_minorProtocolVersion ?
-				WithExisting::Verify
-			:
-				WithExisting::Trust;
-	}
-	catch (...)
-	{
-		m_action = WithExisting::Kill;
-	}
+	_out << "Since " << toString(_r.since) << " (" << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - _r.since).count();
+	_out << "): " << _r.ticks << "ticks";
+	return _out;
 }
 
-void VersionChecker::setOk()
+#ifdef _WIN32
+const char* ClientNote::name() { return EthTeal "^" EthBlue " i"; }
+const char* ClientChat::name() { return EthTeal "^" EthWhite " o"; }
+const char* ClientTrace::name() { return EthTeal "^" EthGray " O"; }
+const char* ClientDetail::name() { return EthTeal "^" EthCoal " 0"; }
+#else
+const char* ClientNote::name() { return EthTeal "⧫" EthBlue " ℹ"; }
+const char* ClientChat::name() { return EthTeal "⧫" EthWhite " ◌"; }
+const char* ClientTrace::name() { return EthTeal "⧫" EthGray " ◎"; }
+const char* ClientDetail::name() { return EthTeal "⧫" EthCoal " ●"; }
+#endif
+
+static const Addresses c_canaries =
 {
-	if (m_action != WithExisting::Trust)
-	{
-		try
-		{
-			boost::filesystem::create_directory(m_path);
-		}
-		catch (...)
-		{
-			cwarn << "Unhandled exception! Failed to create directory: " << m_path << "\n" << boost::current_exception_diagnostic_information();
-		}
-		writeFile(m_path + "/status", rlpList(eth::c_protocolVersion, eth::c_minorProtocolVersion, c_databaseVersion, CanonBlockChain::genesis().hash()));
-	}
+	Address("4bb7e8ae99b645c2b7860b8f3a2328aae28bd80a"),		// gav
+	Address("1baf27b88c48dd02b744999cf3522766929d2b2a"),		// vitalik
+	Address("a8edb1ac2c86d3d9d78f96cd18001f60df29e52c"),		// jeff
+	Address("60d11b58744784dc97f878f7e3749c0f1381a004")			// christoph
+};
+
+VersionChecker::VersionChecker(string const& _dbPath)
+{
+	upgradeDatabase(_dbPath);
+}
+
+Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId):
+	Client(_extNet, make_shared<TrivialGasPricer>(), _dbPath, _forceAction, _networkId)
+{
+	startWorking();
+}
+
+Client::Client(p2p::Host* _extNet, std::shared_ptr<GasPricer> _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId):
+	Worker("eth", 0),
+	m_vc(_dbPath),
+	m_bc(_dbPath, _forceAction, [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }),
+	m_gp(_gp),
+	m_stateDB(State::openDB(_dbPath, _forceAction)),
+	m_preMine(m_stateDB, BaseState::CanonGenesis),
+	m_postMine(m_stateDB)
+{
+	m_lastGetWork = std::chrono::system_clock::now() - chrono::seconds(30);
+	m_tqReady = m_tq.onReady([=](){ this->onTransactionQueueReady(); });	// TODO: should read m_tq->onReady(thisThread, syncTransactionQueue);
+	m_bqReady = m_bq.onReady([=](){ this->onBlockQueueReady(); });			// TODO: should read m_bq->onReady(thisThread, syncBlockQueue);
+	m_bq.setOnBad([=](Exception& ex){ this->onBadBlock(ex); });
+	m_bc.setOnBad([=](Exception& ex){ this->onBadBlock(ex); });
+	m_farm.onSolutionFound([=](ProofOfWork::Solution const& s){ return this->submitWork(s); });
+
+	m_gp->update(m_bc);
+
+	auto host = _extNet->registerCapability(new EthereumHost(m_bc, m_tq, m_bq, _networkId));
+	m_host = host;
+	_extNet->addCapability(host, EthereumHost::staticName(), EthereumHost::c_oldProtocolVersion); //TODO: remove this one v61+ protocol is common
+
+	if (_dbPath.size())
+		Defaults::setDBPath(_dbPath);
+	doWork();
+
+	startWorking();
+}
+
+Client::~Client()
+{
+	stopWorking();
 }
 
 ImportResult Client::queueBlock(bytes const& _block, bool _isSafe)
@@ -211,142 +239,6 @@ void Client::onBadBlock(Exception& _ex) const
 #endif
 }
 
-void BasicGasPricer::update(BlockChain const& _bc)
-{
-	unsigned c = 0;
-	h256 p = _bc.currentHash();
-	m_gasPerBlock = _bc.info(p).gasLimit;
-
-	map<u256, u256> dist;
-	u256 total = 0;
-
-	// make gasPrice versus gasUsed distribution for the last 1000 blocks
-	while (c < 1000 && p)
-	{
-		BlockInfo bi = _bc.info(p);
-		if (bi.transactionsRoot != EmptyTrie)
-		{
-			auto bb = _bc.block(p);
-			RLP r(bb);
-			BlockReceipts brs(_bc.receipts(bi.hash()));
-			size_t i = 0;
-			for (auto const& tr: r[1])
-			{
-				Transaction tx(tr.data(), CheckTransaction::None);
-				u256 gu = brs.receipts[i].gasUsed();
-				dist[tx.gasPrice()] += gu;
-				total += gu;
-				i++;
-			}
-		}
-		p = bi.parentHash;
-		++c;
-	}
-
-	// fill m_octiles with weighted gasPrices
-	if (total > 0)
-	{
-		m_octiles[0] = dist.begin()->first;
-
-		// calc mean
-		u256 mean = 0;
-		for (auto const& i: dist)
-			mean += i.first * i.second;
-		mean /= total;
-
-		// calc standard deviation
-		u256 sdSquared = 0;
-		for (auto const& i: dist)
-			sdSquared += i.second * (i.first - mean) * (i.first - mean);
-		sdSquared /= total;
-
-		if (sdSquared)
-		{
-			long double sd = sqrt(sdSquared.convert_to<long double>());
-			long double normalizedSd = sd / mean.convert_to<long double>();
-
-			// calc octiles normalized to gaussian distribution
-			boost::math::normal gauss(1.0, (normalizedSd > 0.01) ? normalizedSd : 0.01);
-			for (size_t i = 1; i < 8; i++)
-				m_octiles[i] = u256(mean.convert_to<long double>() * boost::math::quantile(gauss, i / 8.0));
-			m_octiles[8] = dist.rbegin()->first;
-		}
-		else
-		{
-			for (size_t i = 0; i < 9; i++)
-				m_octiles[i] = (i + 1) * mean / 5;
-		}
-	}
-}
-
-std::ostream& dev::eth::operator<<(std::ostream& _out, ActivityReport const& _r)
-{
-	_out << "Since " << toString(_r.since) << " (" << std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - _r.since).count();
-	_out << "): " << _r.ticks << "ticks";
-	return _out;
-}
-
-#ifdef _WIN32
-const char* ClientNote::name() { return EthTeal "^" EthBlue " i"; }
-const char* ClientChat::name() { return EthTeal "^" EthWhite " o"; }
-const char* ClientTrace::name() { return EthTeal "^" EthGray " O"; }
-const char* ClientDetail::name() { return EthTeal "^" EthCoal " 0"; }
-#else
-const char* ClientNote::name() { return EthTeal "⧫" EthBlue " ℹ"; }
-const char* ClientChat::name() { return EthTeal "⧫" EthWhite " ◌"; }
-const char* ClientTrace::name() { return EthTeal "⧫" EthGray " ◎"; }
-const char* ClientDetail::name() { return EthTeal "⧫" EthCoal " ●"; }
-#endif
-
-Client::Client(p2p::Host* _extNet, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId):
-	Client(_extNet, make_shared<TrivialGasPricer>(), _dbPath, _forceAction, _networkId)
-{
-	startWorking();
-}
-
-Client::Client(p2p::Host* _extNet, std::shared_ptr<GasPricer> _gp, std::string const& _dbPath, WithExisting _forceAction, u256 _networkId):
-	Worker("eth", 0),
-	m_vc(_dbPath),
-	m_bc(_dbPath, max(m_vc.action(), _forceAction), [](unsigned d, unsigned t){ cerr << "REVISING BLOCKCHAIN: Processed " << d << " of " << t << "...\r"; }),
-	m_gp(_gp),
-	m_stateDB(State::openDB(_dbPath, max(m_vc.action(), _forceAction))),
-	m_preMine(m_stateDB, BaseState::CanonGenesis),
-	m_postMine(m_stateDB)
-{
-	m_lastGetWork = std::chrono::system_clock::now() - chrono::seconds(30);
-	m_tqReady = m_tq.onReady([=](){ this->onTransactionQueueReady(); });	// TODO: should read m_tq->onReady(thisThread, syncTransactionQueue);
-	m_bqReady = m_bq.onReady([=](){ this->onBlockQueueReady(); });			// TODO: should read m_bq->onReady(thisThread, syncBlockQueue);
-	m_bq.setOnBad([=](Exception& ex){ this->onBadBlock(ex); });
-	m_bc.setOnBad([=](Exception& ex){ this->onBadBlock(ex); });
-	m_farm.onSolutionFound([=](ProofOfWork::Solution const& s){ return this->submitWork(s); });
-
-	m_gp->update(m_bc);
-
-	auto host = _extNet->registerCapability(new EthereumHost(m_bc, m_tq, m_bq, _networkId));
-	m_host = host;
-	_extNet->addCapability(host, EthereumHost::staticName(), EthereumHost::c_oldProtocolVersion); //TODO: remove this one v61+ protocol is common
-
-	if (_dbPath.size())
-		Defaults::setDBPath(_dbPath);
-	m_vc.setOk();
-	doWork();
-
-	startWorking();
-}
-
-Client::~Client()
-{
-	stopWorking();
-}
-
-static const Addresses c_canaries =
-{
-	Address("4bb7e8ae99b645c2b7860b8f3a2328aae28bd80a"),		// gav
-	Address("1baf27b88c48dd02b744999cf3522766929d2b2a"),		// vitalik
-	Address("a8edb1ac2c86d3d9d78f96cd18001f60df29e52c"),		// jeff
-	Address("60d11b58744784dc97f878f7e3749c0f1381a004")			// christoph
-};
-
 bool Client::isChainBad() const
 {
 	unsigned numberBad = 0;

libethereum/Client.h

@@ -64,33 +64,6 @@ class VersionChecker
 {
 public:
 	VersionChecker(std::string const& _dbPath);
-
-	void setOk();
-	WithExisting action() const { return m_action; }
-
-private:
-	WithExisting m_action;
-	std::string m_path;
-};
-
-class BasicGasPricer: public GasPricer
-{
-public:
-	explicit BasicGasPricer(u256 _weiPerRef, u256 _refsPerBlock): m_weiPerRef(_weiPerRef), m_refsPerBlock(_refsPerBlock) {}
-
-	void setRefPrice(u256 _weiPerRef) { if ((bigint)m_refsPerBlock * _weiPerRef > std::numeric_limits<u256>::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_weiPerRef = _weiPerRef; }
-	void setRefBlockFees(u256 _refsPerBlock) { if ((bigint)m_weiPerRef * _refsPerBlock > std::numeric_limits<u256>::max() ) BOOST_THROW_EXCEPTION(Overflow() << errinfo_comment("ether price * block fees is larger than 2**256-1, choose a smaller number.") ); else m_refsPerBlock = _refsPerBlock; }
-
-	u256 ask(State const&) const override { return m_weiPerRef * m_refsPerBlock / m_gasPerBlock; }
-	u256 bid(TransactionPriority _p = TransactionPriority::Medium) const override { return m_octiles[(int)_p] > 0 ? m_octiles[(int)_p] : (m_weiPerRef * m_refsPerBlock / m_gasPerBlock); }
-
-	void update(BlockChain const& _bc) override;
-
-private:
-	u256 m_weiPerRef;
-	u256 m_refsPerBlock;
-	u256 m_gasPerBlock = 3141592;
-	std::array<u256, 9> m_octiles;
 };
 
 struct ClientNote: public LogChannel { static const char* name(); static const int verbosity = 2; };

libethereum/EthereumHost.cpp

@@ -286,7 +286,7 @@ void EthereumHost::onPeerTransactions(std::shared_ptr<EthereumPeer> _peer, RLP c
 	unsigned itemCount = _r.itemCount();
 	clog(NetAllDetail) << "Transactions (" << dec << itemCount << "entries)";
 	Guard l(_peer->x_knownTransactions);
-	for (unsigned i = 0; i < min<unsigned>(itemCount, 256); ++i)	// process 256 transactions at most. TODO: much better solution.
+	for (unsigned i = 0; i < min<unsigned>(itemCount, 32); ++i)	// process 256 transactions at most. TODO: much better solution.
 	{
 		auto h = sha3(_r[i].data());
 		_peer->m_knownTransactions.insert(h);

libethereum/GasPricer.cpp

@@ -0,0 +1,26 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file GasPricer.cpp
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2015
+ */
+
+#include "GasPricer.h"
+
+using namespace std;
+using namespace dev;
+using namespace dev::eth;

libethereum/GasPricer.h

@@ -0,0 +1,74 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+/** @file GasPricer.h
+ * @author Gav Wood <i@gavwood.com>
+ * @date 2014
+ */
+
+#pragma once
+
+#include <libethcore/Common.h>
+
+namespace dev
+{
+namespace eth
+{
+
+class State;
+class BlockChain;
+
+enum class TransactionPriority
+{
+	Lowest = 0,
+	Low = 2,
+	Medium = 4,
+	High = 6,
+	Highest = 8
+};
+
+class GasPricer
+{
+public:
+	GasPricer() = default;
+	virtual ~GasPricer() = default;
+
+	virtual u256 ask(State const&) const = 0;
+	virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0;
+
+	virtual void update(BlockChain const&) {}
+};
+
+class TrivialGasPricer: public GasPricer
+{
+public:
+	TrivialGasPricer() = default;
+	TrivialGasPricer(u256 const& _ask, u256 const& _bid): m_ask(_ask), m_bid(_bid) {}
+
+	void setAsk(u256 const& _ask) { m_ask = _ask; }
+	void setBid(u256 const& _bid) { m_bid = _bid; }
+
+	u256 ask() const { return m_ask; }
+	u256 ask(State const&) const override { return m_ask; }
+	u256 bid(TransactionPriority = TransactionPriority::Medium) const override { return m_bid; }
+
+private:
+	u256 m_ask = 10 * szabo;
+	u256 m_bid = 10 * szabo;
+};
+
+}
+}

libethereum/State.cpp

@@ -41,6 +41,7 @@
 using namespace std;
 using namespace dev;
 using namespace dev::eth;
+namespace fs = boost::filesystem;
 
 #define ctrace clog(StateTrace)
 #define ETH_TIMED_ENACTMENTS 0
@@ -52,23 +53,27 @@ const char* StateDetail::name() { return EthViolet "⚙" EthWhite " ◌"; }
 const char* StateTrace::name() { return EthViolet "⚙" EthGray " ◎"; }
 const char* StateChat::name() { return EthViolet "⚙" EthWhite " ◌"; }
 
-OverlayDB State::openDB(std::string _path, WithExisting _we)
+OverlayDB State::openDB(std::string const& _basePath, WithExisting _we)
 {
-	if (_path.empty())
-		_path = Defaults::get()->m_dbPath;
-	boost::filesystem::create_directory(_path);
+	std::string path = _basePath.empty() ? Defaults::get()->m_dbPath : _basePath;
 
 	if (_we == WithExisting::Kill)
-		boost::filesystem::remove_all(_path + "/state");
+	{
+		cnote << "Killing state database (WithExisting::Kill).";
+		boost::filesystem::remove_all(path + "/state");
+	}
+
+	path += "/" + toHex(CanonBlockChain::genesis().hash().ref().cropped(0, 4)) + "/" + toString(c_databaseVersion);
+	boost::filesystem::create_directory(path);
 
 	ldb::Options o;
 	o.max_open_files = 256;
 	o.create_if_missing = true;
 	ldb::DB* db = nullptr;
-	ldb::DB::Open(o, _path + "/state", &db);
+	ldb::DB::Open(o, path + "/state", &db);
 	if (!db)
 	{
-		if (boost::filesystem::space(_path + "/state").available < 1024)
+		if (boost::filesystem::space(path + "/state").available < 1024)
 		{
 			cwarn << "Not enough available space found on hard drive. Please free some up and then re-run. Bailing.";
 			BOOST_THROW_EXCEPTION(NotEnoughAvailableSpace());

libethereum/State.h

@@ -37,6 +37,7 @@
 #include "Transaction.h"
 #include "TransactionReceipt.h"
 #include "AccountDiff.h"
+#include "GasPricer.h"
 
 namespace dev
 {
@@ -80,45 +81,6 @@ enum class BaseState
 	CanonGenesis
 };
 
-enum class TransactionPriority
-{
-	Lowest = 0,
-	Low = 2,
-	Medium = 4,
-	High = 6,
-	Highest = 8
-};
-
-class GasPricer
-{
-public:
-	GasPricer() = default;
-	virtual ~GasPricer() = default;
-
-	virtual u256 ask(State const&) const = 0;
-	virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0;
-
-	virtual void update(BlockChain const&) {}
-};
-
-class TrivialGasPricer: public GasPricer
-{
-public:
-	TrivialGasPricer() = default;
-	TrivialGasPricer(u256 const& _ask, u256 const& _bid): m_ask(_ask), m_bid(_bid) {}
-
-	void setAsk(u256 const& _ask) { m_ask = _ask; }
-	void setBid(u256 const& _bid) { m_bid = _bid; }
-
-	u256 ask() const { return m_ask; }
-	u256 ask(State const&) const override { return m_ask; }
-	u256 bid(TransactionPriority = TransactionPriority::Medium) const override { return m_bid; }
-
-private:
-	u256 m_ask = 10 * szabo;
-	u256 m_bid = 10 * szabo;
-};
-
 enum class Permanence
 {
 	Reverted,
@@ -171,7 +133,7 @@ public:
 	Address address() const { return m_ourAddress; }
 
 	/// Open a DB - useful for passing into the constructor & keeping for other states that are necessary.
-	static OverlayDB openDB(std::string _path, WithExisting _we = WithExisting::Trust);
+	static OverlayDB openDB(std::string const& _path, WithExisting _we = WithExisting::Trust);
 	static OverlayDB openDB(WithExisting _we = WithExisting::Trust) { return openDB(std::string(), _we); }
 	OverlayDB const& db() const { return m_db; }
 	OverlayDB& db() { return m_db; }

libethereum/TransactionQueue.cpp

@@ -98,11 +98,7 @@ ImportResult TransactionQueue::import(Transaction const& _transaction, ImportCal
 std::unordered_map<h256, Transaction> TransactionQueue::transactions() const
 {
 	ReadGuard l(m_lock);
-	auto ret = m_current;
-	for (auto const& i: m_future)
-		if (i.second.nonce() < maxNonce_WITH_LOCK(i.second.sender()))
-			ret.insert(i);
-	return ret;
+	return m_current + m_future;
 }
 
 ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transaction const& _transaction, ImportCallback const& _cb)
@@ -113,6 +109,9 @@ ImportResult TransactionQueue::manageImport_WITH_LOCK(h256 const& _h, Transactio
 		// If it doesn't work, the signature is bad.
 		// The transaction's nonce may yet be invalid (or, it could be "valid" but we may be missing a marginally older transaction).
 
+
+		// Remove any prior transaction with the same nonce but a lower gas price.
+		// Bomb out if there's a prior transaction with higher gas price.
 		auto r = m_senders.equal_range(_transaction.from());
 		for (auto it = r.first; it != r.second; ++it)
 			if (m_current.count(it->second) && m_current[it->second].nonce() == _transaction.nonce())

libethereum/Utility.cpp

@@ -22,11 +22,16 @@
 #include "Utility.h"
 
 #include <boost/regex.hpp>
-#include <libethcore/Common.h>
+#include <boost/filesystem.hpp>
 #include <libdevcore/SHA3.h>
+#include <libdevcore/RLP.h>
+#include <libdevcore/Log.h>
+#include <libethcore/Common.h>
+#include "Defaults.h"
 using namespace std;
 using namespace dev;
 using namespace dev::eth;
+namespace fs = boost::filesystem;
 
 bytes dev::eth::parseData(string const& _args)
 {
@@ -84,3 +89,47 @@ bytes dev::eth::parseData(string const& _args)
 
 	return m_data;
 }
+
+void dev::eth::upgradeDatabase(std::string const& _basePath)
+{
+	std::string path = _basePath.empty() ? Defaults::get()->dbPath() : _basePath;
+
+	if (fs::exists(path + "/state") && fs::exists(path + "/details") && fs::exists(path + "/blocks"))
+	{
+		// upgrade
+		cnote << "Upgrading database to new layout...";
+		bytes statusBytes = contents(path + "/status");
+		RLP status(statusBytes);
+		try
+		{
+			auto minorProtocolVersion = (unsigned)status[1];
+			auto databaseVersion = (unsigned)status[2];
+			auto genesisHash = (h256)status[3];
+
+			string chainPath = path + "/" + toHex(genesisHash.ref().cropped(0, 4));
+			string extrasPath = chainPath + "/" + toString(databaseVersion);
+
+			// write status
+			if (!fs::exists(chainPath + "/blocks"))
+			{
+				boost::filesystem::create_directories(chainPath);
+				fs::rename(path + "/blocks", chainPath + "/blocks");
+
+				if (!fs::exists(extrasPath + "/extras"))
+				{
+					boost::filesystem::create_directories(extrasPath);
+					fs::rename(path + "/details", extrasPath + "/extras");
+					fs::rename(path + "/state", extrasPath + "/state");
+					writeFile(extrasPath + "/minor", rlp(minorProtocolVersion));
+
+					fs::remove_all(path + "/status");
+				}
+			}
+		}
+		catch (...)
+		{
+			cwarn << "Couldn't upgrade - bad status";
+		}
+	}
+}
+

libethereum/Utility.h

@@ -42,5 +42,7 @@ namespace eth
  */
 bytes parseData(std::string const& _args);
 
+void upgradeDatabase(std::string const& _basePath);
+
 }
 }