Basic paranoia compiler flag.

Additional fixes & cleanups for ref counting.

CMakeLists.txt

@@ -11,6 +11,18 @@ if ("x${HEADLESS}" STREQUAL "x")
 	set(HEADLESS 0)
 endif ()
 
+if ("x${PARANOIA}" STREQUAL "x")
+    if ("x${CMAKE_BUILD_TYPE}" STREQUAL "xDebug")
+        set(PARANOIA 1)
+    else ()
+        set(PARANOIA 0)
+    endif ()
+endif ()
+
+if (PARANOIA)
+    add_definitions(-DETH_PARANOIA)
+endif ()
+
 # Default TARGET_PLATFORM to "linux".
 set(TARGET_PLATFORM CACHE STRING "linux")
 if ("x${TARGET_PLATFORM}" STREQUAL "x")
@@ -250,6 +262,7 @@ add_subdirectory(eth)
 if (NOT ("${TARGET_PLATFORM}" STREQUAL "w64"))
 	add_subdirectory(neth)
 endif ()
+
 if (NOT HEADLESS)
 	add_subdirectory(libqethereum)
 	add_subdirectory(alethzero)

libethcore/Log.cpp

@@ -27,7 +27,7 @@ using namespace std;
 using namespace eth;
 
 // Logging
-int eth::g_logVerbosity = 8;
+int eth::g_logVerbosity = 7;
 map<type_info const*, bool> eth::g_logOverride;
 
 ThreadLocalLogName eth::t_logThreadName("main");

libethcore/TrieDB.cpp

@@ -24,8 +24,6 @@
 using namespace std;
 using namespace eth;
 
-#define tdebug ndebug
-
 namespace eth
 {
 
@@ -38,8 +36,8 @@ std::string BasicMap::lookup(h256 _h) const
 	{
 		if (!m_enforceRefs || (m_refCount.count(it->first) && m_refCount.at(it->first)))
 			return it->second;
-		else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first))
-			cwarn << "XXX Lookup required for value with no refs:" << _h.abridged();
+//		else if (m_enforceRefs && m_refCount.count(it->first) && !m_refCount.at(it->first))
+//			cnote << "Lookup required for value with no refs. Let's hope it's in the DB." << _h.abridged();
 	}
 	return std::string();
 }
@@ -59,16 +57,27 @@ void BasicMap::insert(h256 _h, bytesConstRef _v)
 	tdebug << "INST" << _h.abridged() << "=>" << m_refCount[_h];
 }
 
-void BasicMap::kill(h256 _h)
+bool BasicMap::kill(h256 _h)
 {
 	if (m_refCount.count(_h))
 	{
 		if (m_refCount[_h] > 0)
 			--m_refCount[_h];
 		else
-			cwarn << "Decreasing DB node ref count below zero. Probably have a corrupt Trie." << _h.abridged();
+		{
+			// If we get to this point, then there was probably a node in the level DB which we need to remove and which we have previously
+			// used as part of the memory-based BasicMap. Nothing to be worried about *as long as the node exists in the DB*.
+			tdebug << "NOKILL-WAS" << _h.abridged();
+			return false;
+		}
+		tdebug << "KILL" << _h.abridged() << "=>" << m_refCount[_h];
+		return true;
+	}
+	else
+	{
+		tdebug << "NOKILL" << _h.abridged();
+		return false;
 	}
-	tdebug << "KILL" << _h.abridged() << "=>" << m_refCount[_h];
 }
 
 void BasicMap::purge()
@@ -131,4 +140,18 @@ bool Overlay::exists(h256 _h) const
 	return !ret.empty();
 }
 
+void Overlay::kill(h256 _h)
+{
+	if (!BasicMap::kill(_h))
+	{
+#if ETH_PARANOIA
+		std::string ret;
+		if (m_db)
+			m_db->Get(m_readOptions, ldb::Slice((char const*)_h.data(), 32), &ret);
+		if (ret.empty())
+			cnote << "Decreasing DB node ref count below zero with no DB node. Probably have a corrupt Trie." << _h.abridged();
+#endif
+	}
+}
+
 }

libethcore/TrieDB.h

@@ -28,11 +28,13 @@
 #include "TrieCommon.h"
 namespace ldb = leveldb;
 
-#define tdebug ndebug
-
 namespace eth
 {
 
+struct TrieDBChannel: public LogChannel  { static const char* name() { return "-T-"; } static const int verbosity = 6; };
+
+#define tdebug clog(TrieDBChannel)
+
 class BasicMap
 {
 	friend class EnforceRefs;
@@ -46,7 +48,7 @@ public:
 	std::string lookup(h256 _h) const;
 	bool exists(h256 _h) const;
 	void insert(h256 _h, bytesConstRef _v);
-	void kill(h256 _h);
+	bool kill(h256 _h);
 	void purge();
 
 protected:
@@ -85,6 +87,7 @@ public:
 
 	std::string lookup(h256 _h) const;
 	bool exists(h256 _h) const;
+	void kill(h256 _h);
 
 private:
 	using BasicMap::clear;
@@ -320,7 +323,7 @@ private:
 	std::string atAux(RLP const& _here, NibbleSlice _key) const;
 
 	void mergeAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key, bytesConstRef _value);
-	bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v);
+	bytes mergeAt(RLP const& _replace, NibbleSlice _k, bytesConstRef _v, bool _inline = false);
 
 	bool deleteAtAux(RLPStream& _out, RLP const& _replace, NibbleSlice _key);
 	bytes deleteAt(RLP const& _replace, NibbleSlice _k);
@@ -448,7 +451,7 @@ template <class DB> void GenericTrieDB<DB>::init()
 
 template <class DB> void GenericTrieDB<DB>::insert(bytesConstRef _key, bytesConstRef _value)
 {
-	ndebug << "Insert" << toHex(_key.cropped(0, 4));
+//	tdebug << "Insert" << toHex(_key.cropped(0, 4)) << "=>" << toHex(_value);
 
 	std::string rv = node(m_root);
 	assert(rv.size());
@@ -498,7 +501,7 @@ template <class DB> std::string GenericTrieDB<DB>::atAux(RLP const& _here, Nibbl
 	}
 }
 
-template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
+template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSlice _k, bytesConstRef _v, bool _inline)
 {
 	tdebug << "mergeAt " << _orig << _k << sha3(_orig.data()).abridged();
 
@@ -523,7 +526,8 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 		// partial key is our key - move down.
 		if (_k.contains(k) && !isLeaf(_orig))
 		{
-			killNode(sha3(_orig.data()));
+			if (!_inline)
+				killNode(_orig);
 			RLPStream s(2);
 			s.append(_orig[0]);
 			mergeAtAux(s, _orig[1], _k.mid(k.size()), _v);
@@ -534,10 +538,10 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 //		std::cout << _k << " sh " << k << " = " << sh << std::endl;
 		if (sh)
 			// shared stuff - cleve at disagreement.
-			return mergeAt(RLP(cleve(_orig, sh)), _k, _v);
+			return mergeAt(RLP(cleve(_orig, sh)), _k, _v, true);
 		else
 			// nothing shared - branch
-			return mergeAt(RLP(branch(_orig)), _k, _v);
+			return mergeAt(RLP(branch(_orig)), _k, _v, true);
 	}
 	else
 	{
@@ -548,7 +552,8 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 			return place(_orig, _k, _v);
 
 		// Kill the node.
-		killNode(sha3(_orig.data()));
+		if (!_inline)
+			killNode(_orig);
 
 		// not exactly our node - delve to next level at the correct index.
 		byte n = _k[0];
@@ -565,16 +570,19 @@ template <class DB> bytes GenericTrieDB<DB>::mergeAt(RLP const& _orig, NibbleSli
 
 template <class DB> void GenericTrieDB<DB>::mergeAtAux(RLPStream& _out, RLP const& _orig, NibbleSlice _k, bytesConstRef _v)
 {
-	tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()).abridged() << _orig.toHash<h256>().abridged();
+	tdebug << "mergeAtAux " << _orig << _k << sha3(_orig.data()).abridged() << ((_orig.isData() && _orig.size() <= 32) ? _orig.toHash<h256>().abridged() : std::string());
 	RLP r = _orig;
 	std::string s;
+	// _orig is always a segment of a node's RLP - removing it alone is pointless. However, if may be a hash, in which case we deref and we know it is removable.
+	bool isRemovable = false;
 	if (!r.isList() && !r.isEmpty())
 	{
 		s = node(_orig.toHash<h256>());
 		r = RLP(s);
 		assert(!r.isNull());
+		isRemovable = true;
 	}
-	bytes b = mergeAt(r, _k, _v);
+	bytes b = mergeAt(r, _k, _v, !isRemovable);
 	streamNode(_out, b);
 }
 
@@ -630,7 +638,7 @@ template <class DB> bytes GenericTrieDB<DB>::deleteAt(RLP const& _orig, NibbleSl
 			s.appendList(2) << _orig[0];
 			if (!deleteAtAux(s, _orig[1], _k.mid(k.size())))
 				return bytes();
-			killNode(sha3(_orig.data()));
+			killNode(_orig);
 			RLP r(s.out());
 			if (isTwoItemNode(r[1]))
 				return graft(r);
@@ -648,7 +656,7 @@ template <class DB> bytes GenericTrieDB<DB>::deleteAt(RLP const& _orig, NibbleSl
 		if (_k.size() == 0 && !_orig[16].isEmpty())
 		{
 			// Kill the node.
-			killNode(sha3(_orig.data()));
+			killNode(_orig);
 
 			byte used = uniqueInUse(_orig, 16);
 			if (used != 255)

libethereum/Executive.cpp

@@ -108,11 +108,6 @@ void Executive::call(Address _receiveAddress, Address _senderAddress, u256 _valu
 
 void Executive::create(Address _sender, u256 _endowment, u256 _gasPrice, u256 _gas, bytesConstRef _init, Address _origin)
 {
-#if ETH_PARANOIA
-
-
-#endif
-
 	m_newAddress = right160(sha3(rlpList(_sender, m_s.transactionsFrom(_sender) - 1)));
 	while (m_s.addressInUse(m_newAddress))
 		m_newAddress = (u160)m_newAddress + 1;