Merge pull request #1312 from imapp-pl/feature/testeth_stats

Stats for testeth + bonus

libdevcrypto/Common.h

@@ -45,7 +45,7 @@ using Signature = h520;
 
 struct SignatureStruct
 {
-	SignatureStruct() {}
+	SignatureStruct() = default;
 	SignatureStruct(Signature const& _s) { *(h520*)this = _s; }
 	SignatureStruct(h256 const& _r, h256 const& _s, byte _v): r(_r), s(_s), v(_v) {}
 	operator Signature() const { return *(h520 const*)this; }
@@ -55,7 +55,7 @@ struct SignatureStruct
 
 	h256 r;
 	h256 s;
-	byte v;
+	byte v = 0;
 };
 
 /// An Ethereum address: 20 bytes.

libethereum/State.h

@@ -68,7 +68,8 @@ enum class TransactionPriority
 class GasPricer
 {
 public:
-	GasPricer() {}
+	GasPricer() = default;
+	virtual ~GasPricer() = default;
 
 	virtual u256 ask(State const&) const = 0;
 	virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0;

test/CMakeLists.txt

@@ -19,10 +19,10 @@ include_directories(${JSON_RPC_CPP_INCLUDE_DIRS})
 
 file(GLOB HEADERS "*.h")
 add_executable(testeth ${SRC_LIST} ${HEADERS})
-add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp)
+add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
-add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp)
+add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp Stats.cpp)
-add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp)
+add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
-add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp)
+add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp Stats.cpp)
 
 target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
 target_link_libraries(testeth ${CURL_LIBRARIES})

test/Stats.cpp

@@ -0,0 +1,83 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "Stats.h"
+
+#include <iterator>
+#include <numeric>
+
+namespace dev
+{
+namespace test
+{
+
+Stats& Stats::get()
+{
+	static Stats instance;
+	return instance;
+}
+
+void Stats::testStarted(std::string const& _name)
+{
+	m_currentTest = _name;
+	m_tp = clock::now();
+}
+
+void Stats::testFinished()
+{
+	m_stats[clock::now() - m_tp] = std::move(m_currentTest);
+}
+
+std::ostream& operator<<(std::ostream& out, Stats::clock::duration const& d)
+{
+	return out << std::setw(10) << std::right << std::chrono::duration_cast<std::chrono::microseconds>(d).count() << " us";
+}
+
+Stats::~Stats()
+{
+	if (m_stats.empty())
+		return;
+
+	auto& out = std::cout;
+	auto itr = m_stats.begin();
+	auto min = *itr;
+	auto max = *m_stats.rbegin();
+	std::advance(itr, m_stats.size() / 2);
+	auto med = *itr;
+	auto tot = std::accumulate(m_stats.begin(), m_stats.end(), clock::duration{}, [](clock::duration const& a, stats_t::value_type const& v)
+	{
+		return a + v.first;
+	});
+
+	out << "\nSTATS:\n\n" << std::setfill(' ');
+
+	if (Options::get().statsFull)
+	{
+		for (auto&& s: m_stats)
+			out << "  " << std::setw(40) << std::left << s.second.substr(0, 40) << s.first << " \n";
+		out << "\n";
+	}
+
+	out	<< "  tot: " << tot << "\n"
+		<< "  avg: " << (tot / m_stats.size()) << "\n\n"
+		<< "  min: " << min.first << " (" << min.second << ")\n"
+		<< "  med: " << med.first << " (" << med.second << ")\n"
+		<< "  max: " << max.first << " (" << max.second << ")\n";
+}
+
+}
+}

test/Stats.h

@@ -0,0 +1,50 @@
+/*
+	This file is part of cpp-ethereum.
+
+	cpp-ethereum is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	cpp-ethereum is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with cpp-ethereum.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include <chrono>
+#include <map>
+
+#include "TestHelper.h"
+
+namespace dev
+{
+namespace test
+{
+
+class Stats: public Listener
+{
+public:
+	using clock = std::chrono::high_resolution_clock;
+	using stats_t = std::map<clock::duration, std::string>;
+
+	static Stats& get();
+
+	~Stats();
+
+	void testStarted(std::string const& _name) override;
+	void testFinished() override;
+
+private:
+	clock::time_point m_tp;
+	std::string m_currentTest;
+	stats_t m_stats;
+};
+
+}
+}

test/TestHelper.cpp

@@ -473,7 +473,7 @@ void executeTests(const string& _name, const string& _testPathAppendix, std::fun
 
 	try
 	{
-		cnote << "Testing ..." << _name;
+		std::cout << "TEST " << _name << ":\n";
 		json_spirit::mValue v;
 		string s = asString(dev::contents(testPath + "/" + _name + ".json"));
 		BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of " + testPath + "/" + _name + ".json is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");
@@ -551,6 +551,10 @@ Options::Options()
 			vmtrace = true;
 		else if (arg == "--filltests")
 			fillTests = true;
+		else if (arg == "--stats")
+			stats = true;
+		else if (arg == "--stats=full")
+			stats = statsFull = true;
 		else if (arg == "--performance")
 			performance = true;
 		else if (arg == "--quadratic")
@@ -578,6 +582,7 @@ Options const& Options::get()
 	return instance;
 }
 
+
 LastHashes lastHashes(u256 _currentBlockNumber)
 {
 	LastHashes ret;
@@ -586,4 +591,27 @@ LastHashes lastHashes(u256 _currentBlockNumber)
 	return ret;
 }
 
+
+namespace
+{
+	Listener* g_listener;
+}
+
+void Listener::registerListener(Listener& _listener)
+{
+	g_listener = &_listener;
+}
+
+void Listener::notifyTestStarted(std::string const& _name)
+{
+	if (g_listener)
+		g_listener->testStarted(_name);
+}
+
+void Listener::notifyTestFinished()
+{
+	if (g_listener)
+		g_listener->testFinished();
+}
+
 } } // namespaces

test/TestHelper.h

@@ -162,8 +162,9 @@ class Options
 public:
 	bool jit = false;		///< Use JIT
 	bool vmtrace = false;	///< Create EVM execution tracer // TODO: Link with log verbosity?
-	bool showTimes = false;	///< Print test groups execution times
 	bool fillTests = false; ///< Create JSON test files from execution results
+	bool stats = false;		///< Execution time stats
+	bool statsFull = false; ///< Output full stats - execution times for every test
 
 	/// Test selection
 	/// @{
@@ -183,5 +184,32 @@ private:
 	Options(Options const&) = delete;
 };
 
+/// Allows observing test execution process.
+/// This class also provides methods for registering and notifying the listener
+class Listener
+{
+public:
+	virtual ~Listener() = default;
+
+	virtual void testStarted(std::string const& _name) = 0;
+	virtual void testFinished() = 0;
+
+	static void registerListener(Listener& _listener);
+	static void notifyTestStarted(std::string const& _name);
+	static void notifyTestFinished();
+
+	/// Test started/finished notification RAII helper
+	class ExecTimeGuard
+	{
+	public:
+		ExecTimeGuard(std::string const& _testName) { notifyTestStarted(_testName);	}
+		~ExecTimeGuard() { notifyTestFinished(); }
+		ExecTimeGuard(ExecTimeGuard const&) = delete;
+		ExecTimeGuard& operator=(ExecTimeGuard) = delete;
+	};
+};
+
+
+
 }
 }

test/state.cpp

@@ -31,6 +31,7 @@
 #include <libethereum/Defaults.h>
 #include <libevm/VM.h>
 #include "TestHelper.h"
+#include "Stats.h"
 
 using namespace std;
 using namespace json_spirit;
@@ -41,11 +42,12 @@ namespace dev {  namespace test {
 
 void doStateTests(json_spirit::mValue& v, bool _fillin)
 {
-	Options::get(); // process command line options
+	if (Options::get().stats)
+		Listener::registerListener(Stats::get());
 
 	for (auto& i: v.get_obj())
 	{
-		cerr << i.first << endl;
+		std::cout << "  " << i.first << "\n";
 		mObject& o = i.second.get_obj();
 
 		BOOST_REQUIRE(o.count("env") > 0);
@@ -60,16 +62,17 @@ void doStateTests(json_spirit::mValue& v, bool _fillin)
 
 		try
 		{
+			Listener::ExecTimeGuard guard{i.first};
 			theState.execute(lastHashes(importer.m_environment.currentBlock.number), tx, &output);
 		}
 		catch (Exception const& _e)
 		{
-			cnote << "state execution did throw an exception: " << diagnostic_information(_e);
+			cnote << "Exception:\n" << diagnostic_information(_e);
 			theState.commit();
 		}
 		catch (std::exception const& _e)
 		{
-			cnote << "state execution did throw an exception: " << _e.what();
+			cnote << "state execution exception: " << _e.what();
 		}
 
 		if (_fillin)
@@ -178,29 +181,13 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest)
 BOOST_AUTO_TEST_CASE(stQuadraticComplexityTest)
 {
 	if (test::Options::get().quadratic)
-	{
-		auto start = chrono::steady_clock::now();
-
 		dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests);
-
-		auto end = chrono::steady_clock::now();
-		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-	}
 }
 
 BOOST_AUTO_TEST_CASE(stMemoryStressTest)
 {
 	if (test::Options::get().memory)
-	{
-		auto start = chrono::steady_clock::now();
-
 		dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests);
-
-		auto end = chrono::steady_clock::now();
-		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-	}
 }
 
 BOOST_AUTO_TEST_CASE(stSolidityTest)

test/vm.cpp

@@ -20,13 +20,12 @@
  * vm test functions.
  */
 
-#include <chrono>
-
 #include <boost/filesystem.hpp>
 
 #include <libethereum/Executive.h>
 #include <libevm/VMFactory.h>
 #include "vm.h"
+#include "Stats.h"
 
 using namespace std;
 using namespace json_spirit;
@@ -312,11 +311,12 @@ namespace dev { namespace test {
 
 void doVMTests(json_spirit::mValue& v, bool _fillin)
 {
-	Options::get(); // process command line options // TODO: We need to control the main() function
+	if (Options::get().stats)
+		Listener::registerListener(Stats::get());
 
 	for (auto& i: v.get_obj())
 	{
-		cnote << i.first;
+		std::cout << "  " << i.first << "\n";
 		mObject& o = i.second.get_obj();
 
 		BOOST_REQUIRE(o.count("env") > 0);
@@ -340,17 +340,21 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
 		bytes output;
 		u256 gas;
 		bool vmExceptionOccured = false;
-		auto startTime = std::chrono::high_resolution_clock::now();
 		try
 		{
 			auto vm = eth::VMFactory::create(fev.gas);
 			auto vmtrace = Options::get().vmtrace ? fev.simpleTrace() : OnOpFunc{};
-			output = vm->go(fev, vmtrace).toBytes();
+			auto outputRef = bytesConstRef{};
+			{
+				Listener::ExecTimeGuard guard{i.first};
+				outputRef = vm->go(fev, vmtrace);
+			}
+			output = outputRef.toBytes();
 			gas = vm->gas();
 		}
 		catch (VMException const&)
 		{
-			cnote << "Safe VM Exception";
+			std::cout << "    Safe VM Exception\n";
 			vmExceptionOccured = true;
 		}
 		catch (Exception const& _e)
@@ -364,15 +368,6 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
 			BOOST_ERROR("Failed VM Test with Exception: " << _e.what());
 		}
 
-		auto endTime = std::chrono::high_resolution_clock::now();
-		if (Options::get().showTimes)
-		{
-			auto testDuration = endTime - startTime;
-			cnote << "Execution time: "
-				  << std::chrono::duration_cast<std::chrono::milliseconds>(testDuration).count()
-				  << " ms";
-		}
-
 		// delete null entries in storage for the sake of comparison
 
 		for (auto  &a: fev.addresses)
@@ -513,29 +508,13 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest)
 BOOST_AUTO_TEST_CASE(vmPerformanceTest)
 {
 	if (test::Options::get().performance)
-	{
-		auto start = chrono::steady_clock::now();
-
 		dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests);
-
-		auto end = chrono::steady_clock::now();
-		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-	}
 }
 
 BOOST_AUTO_TEST_CASE(vmInputLimitsTest1)
 {
 	if (test::Options::get().inputLimits)
-	{
-		auto start = chrono::steady_clock::now();
-
 		dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests);
-
-		auto end = chrono::steady_clock::now();
-		auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
-		cnote << "test duration: " << duration.count() << " milliseconds.\n";
-	}
 }
 
 BOOST_AUTO_TEST_CASE(vmInputLimitsTest2)
@@ -565,7 +544,7 @@ BOOST_AUTO_TEST_CASE(vmRandom)
 	{
 		try
 		{
-			cnote << "Testing ..." << path.filename();
+			std::cout << "TEST " << path.filename() << "\n";
 			json_spirit::mValue v;
 			string s = asString(dev::contents(path.string()));
 			BOOST_REQUIRE_MESSAGE(s.length() > 0, "Content of " + path.string() + " is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");