Browse Source

Merge pull request #1312 from imapp-pl/feature/testeth_stats

Stats for testeth + bonus
cl-refactor
Gav Wood 10 years ago
parent
commit
b7fe9e1d10
  1. 4
      libdevcrypto/Common.h
  2. 3
      libethereum/State.h
  3. 8
      test/CMakeLists.txt
  4. 83
      test/Stats.cpp
  5. 50
      test/Stats.h
  6. 30
      test/TestHelper.cpp
  7. 30
      test/TestHelper.h
  8. 27
      test/state.cpp
  9. 45
      test/vm.cpp

4
libdevcrypto/Common.h

@ -45,7 +45,7 @@ using Signature = h520;
struct SignatureStruct
{
SignatureStruct() {}
SignatureStruct() = default;
SignatureStruct(Signature const& _s) { *(h520*)this = _s; }
SignatureStruct(h256 const& _r, h256 const& _s, byte _v): r(_r), s(_s), v(_v) {}
operator Signature() const { return *(h520 const*)this; }
@ -55,7 +55,7 @@ struct SignatureStruct
h256 r;
h256 s;
byte v;
byte v = 0;
};
/// An Ethereum address: 20 bytes.

3
libethereum/State.h

@ -68,7 +68,8 @@ enum class TransactionPriority
class GasPricer
{
public:
GasPricer() {}
GasPricer() = default;
virtual ~GasPricer() = default;
virtual u256 ask(State const&) const = 0;
virtual u256 bid(TransactionPriority _p = TransactionPriority::Medium) const = 0;

8
test/CMakeLists.txt

@ -19,10 +19,10 @@ include_directories(${JSON_RPC_CPP_INCLUDE_DIRS})
file(GLOB HEADERS "*.h")
add_executable(testeth ${SRC_LIST} ${HEADERS})
add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp)
add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp)
add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp)
add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp)
add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp Stats.cpp)
add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp Stats.cpp)
target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
target_link_libraries(testeth ${CURL_LIBRARIES})

83
test/Stats.cpp

@ -0,0 +1,83 @@
/*
This file is part of cpp-ethereum.
cpp-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
cpp-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Stats.h"
#include <iterator>
#include <numeric>
namespace dev
{
namespace test
{
Stats& Stats::get()
{
static Stats instance;
return instance;
}
void Stats::testStarted(std::string const& _name)
{
m_currentTest = _name;
m_tp = clock::now();
}
void Stats::testFinished()
{
m_stats[clock::now() - m_tp] = std::move(m_currentTest);
}
std::ostream& operator<<(std::ostream& out, Stats::clock::duration const& d)
{
return out << std::setw(10) << std::right << std::chrono::duration_cast<std::chrono::microseconds>(d).count() << " us";
}
Stats::~Stats()
{
if (m_stats.empty())
return;
auto& out = std::cout;
auto itr = m_stats.begin();
auto min = *itr;
auto max = *m_stats.rbegin();
std::advance(itr, m_stats.size() / 2);
auto med = *itr;
auto tot = std::accumulate(m_stats.begin(), m_stats.end(), clock::duration{}, [](clock::duration const& a, stats_t::value_type const& v)
{
return a + v.first;
});
out << "\nSTATS:\n\n" << std::setfill(' ');
if (Options::get().statsFull)
{
for (auto&& s: m_stats)
out << " " << std::setw(40) << std::left << s.second.substr(0, 40) << s.first << " \n";
out << "\n";
}
out << " tot: " << tot << "\n"
<< " avg: " << (tot / m_stats.size()) << "\n\n"
<< " min: " << min.first << " (" << min.second << ")\n"
<< " med: " << med.first << " (" << med.second << ")\n"
<< " max: " << max.first << " (" << max.second << ")\n";
}
}
}

50
test/Stats.h

@ -0,0 +1,50 @@
/*
This file is part of cpp-ethereum.
cpp-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
cpp-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <chrono>
#include <map>
#include "TestHelper.h"
namespace dev
{
namespace test
{
class Stats: public Listener
{
public:
using clock = std::chrono::high_resolution_clock;
using stats_t = std::map<clock::duration, std::string>;
static Stats& get();
~Stats();
void testStarted(std::string const& _name) override;
void testFinished() override;
private:
clock::time_point m_tp;
std::string m_currentTest;
stats_t m_stats;
};
}
}

30
test/TestHelper.cpp

@ -473,7 +473,7 @@ void executeTests(const string& _name, const string& _testPathAppendix, std::fun
try
{
cnote << "Testing ..." << _name;
std::cout << "TEST " << _name << ":\n";
json_spirit::mValue v;
string s = asString(dev::contents(testPath + "/" + _name + ".json"));
BOOST_REQUIRE_MESSAGE(s.length() > 0, "Contents of " + testPath + "/" + _name + ".json is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");
@ -551,6 +551,10 @@ Options::Options()
vmtrace = true;
else if (arg == "--filltests")
fillTests = true;
else if (arg == "--stats")
stats = true;
else if (arg == "--stats=full")
stats = statsFull = true;
else if (arg == "--performance")
performance = true;
else if (arg == "--quadratic")
@ -578,6 +582,7 @@ Options const& Options::get()
return instance;
}
LastHashes lastHashes(u256 _currentBlockNumber)
{
LastHashes ret;
@ -586,4 +591,27 @@ LastHashes lastHashes(u256 _currentBlockNumber)
return ret;
}
namespace
{
Listener* g_listener;
}
void Listener::registerListener(Listener& _listener)
{
g_listener = &_listener;
}
void Listener::notifyTestStarted(std::string const& _name)
{
if (g_listener)
g_listener->testStarted(_name);
}
void Listener::notifyTestFinished()
{
if (g_listener)
g_listener->testFinished();
}
} } // namespaces

30
test/TestHelper.h

@ -162,8 +162,9 @@ class Options
public:
bool jit = false; ///< Use JIT
bool vmtrace = false; ///< Create EVM execution tracer // TODO: Link with log verbosity?
bool showTimes = false; ///< Print test groups execution times
bool fillTests = false; ///< Create JSON test files from execution results
bool stats = false; ///< Execution time stats
bool statsFull = false; ///< Output full stats - execution times for every test
/// Test selection
/// @{
@ -183,5 +184,32 @@ private:
Options(Options const&) = delete;
};
/// Allows observing test execution process.
/// This class also provides methods for registering and notifying the listener
class Listener
{
public:
virtual ~Listener() = default;
virtual void testStarted(std::string const& _name) = 0;
virtual void testFinished() = 0;
static void registerListener(Listener& _listener);
static void notifyTestStarted(std::string const& _name);
static void notifyTestFinished();
/// Test started/finished notification RAII helper
class ExecTimeGuard
{
public:
ExecTimeGuard(std::string const& _testName) { notifyTestStarted(_testName); }
~ExecTimeGuard() { notifyTestFinished(); }
ExecTimeGuard(ExecTimeGuard const&) = delete;
ExecTimeGuard& operator=(ExecTimeGuard) = delete;
};
};
}
}

27
test/state.cpp

@ -31,6 +31,7 @@
#include <libethereum/Defaults.h>
#include <libevm/VM.h>
#include "TestHelper.h"
#include "Stats.h"
using namespace std;
using namespace json_spirit;
@ -41,11 +42,12 @@ namespace dev { namespace test {
void doStateTests(json_spirit::mValue& v, bool _fillin)
{
Options::get(); // process command line options
if (Options::get().stats)
Listener::registerListener(Stats::get());
for (auto& i: v.get_obj())
{
cerr << i.first << endl;
std::cout << " " << i.first << "\n";
mObject& o = i.second.get_obj();
BOOST_REQUIRE(o.count("env") > 0);
@ -60,16 +62,17 @@ void doStateTests(json_spirit::mValue& v, bool _fillin)
try
{
Listener::ExecTimeGuard guard{i.first};
theState.execute(lastHashes(importer.m_environment.currentBlock.number), tx, &output);
}
catch (Exception const& _e)
{
cnote << "state execution did throw an exception: " << diagnostic_information(_e);
cnote << "Exception:\n" << diagnostic_information(_e);
theState.commit();
}
catch (std::exception const& _e)
{
cnote << "state execution did throw an exception: " << _e.what();
cnote << "state execution exception: " << _e.what();
}
if (_fillin)
@ -178,29 +181,13 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest)
BOOST_AUTO_TEST_CASE(stQuadraticComplexityTest)
{
if (test::Options::get().quadratic)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(stMemoryStressTest)
{
if (test::Options::get().memory)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(stSolidityTest)

45
test/vm.cpp

@ -20,13 +20,12 @@
* vm test functions.
*/
#include <chrono>
#include <boost/filesystem.hpp>
#include <libethereum/Executive.h>
#include <libevm/VMFactory.h>
#include "vm.h"
#include "Stats.h"
using namespace std;
using namespace json_spirit;
@ -312,11 +311,12 @@ namespace dev { namespace test {
void doVMTests(json_spirit::mValue& v, bool _fillin)
{
Options::get(); // process command line options // TODO: We need to control the main() function
if (Options::get().stats)
Listener::registerListener(Stats::get());
for (auto& i: v.get_obj())
{
cnote << i.first;
std::cout << " " << i.first << "\n";
mObject& o = i.second.get_obj();
BOOST_REQUIRE(o.count("env") > 0);
@ -340,17 +340,21 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
bytes output;
u256 gas;
bool vmExceptionOccured = false;
auto startTime = std::chrono::high_resolution_clock::now();
try
{
auto vm = eth::VMFactory::create(fev.gas);
auto vmtrace = Options::get().vmtrace ? fev.simpleTrace() : OnOpFunc{};
output = vm->go(fev, vmtrace).toBytes();
auto outputRef = bytesConstRef{};
{
Listener::ExecTimeGuard guard{i.first};
outputRef = vm->go(fev, vmtrace);
}
output = outputRef.toBytes();
gas = vm->gas();
}
catch (VMException const&)
{
cnote << "Safe VM Exception";
std::cout << " Safe VM Exception\n";
vmExceptionOccured = true;
}
catch (Exception const& _e)
@ -364,15 +368,6 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
BOOST_ERROR("Failed VM Test with Exception: " << _e.what());
}
auto endTime = std::chrono::high_resolution_clock::now();
if (Options::get().showTimes)
{
auto testDuration = endTime - startTime;
cnote << "Execution time: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(testDuration).count()
<< " ms";
}
// delete null entries in storage for the sake of comparison
for (auto &a: fev.addresses)
@ -513,29 +508,13 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest)
BOOST_AUTO_TEST_CASE(vmPerformanceTest)
{
if (test::Options::get().performance)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(vmInputLimitsTest1)
{
if (test::Options::get().inputLimits)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(vmInputLimitsTest2)
@ -565,7 +544,7 @@ BOOST_AUTO_TEST_CASE(vmRandom)
{
try
{
cnote << "Testing ..." << path.filename();
std::cout << "TEST " << path.filename() << "\n";
json_spirit::mValue v;
string s = asString(dev::contents(path.string()));
BOOST_REQUIRE_MESSAGE(s.length() > 0, "Content of " + path.string() + " is empty. Have you cloned the 'tests' repo branch develop and set ETHEREUM_TEST_PATH to its path?");

Loading…
Cancel
Save