lukechilds
/
ethminer
mirror of https://github.com/lukechilds/ethminer.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Stats for testeth 

Simple listener support added to testeth. Stats class implements the Listener interface and collects tests execution times. Try options: --stats or --stats=full.

Closes ethereum/cpp-ethereum#1285
cl-refactor
Paweł Bylica 10 years ago
committed by Paweł Bylica
parent
bb150eae43
commit
9b851e9359
7 changed files with 207 additions and 52 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      test/CMakeLists.txt
  2. 83
      test/Stats.cpp
  3. 50
      test/Stats.h
  4. 28
      test/TestHelper.cpp
  5. 30
      test/TestHelper.h
  6. 21
      test/state.cpp
  7. 39
      test/vm.cpp

8
test/CMakeLists.txt
View File

@ -19,10 +19,10 @@ include_directories(${JSON_RPC_CPP_INCLUDE_DIRS})
file(GLOB HEADERS "*.h")
add_executable(testeth ${SRC_LIST} ${HEADERS})
add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp)
add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp)
add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp)
add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp)
add_executable(createRandomVMTest createRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
add_executable(createRandomStateTest createRandomStateTest.cpp TestHelper.cpp Stats.cpp)
add_executable(checkRandomVMTest checkRandomVMTest.cpp vm.cpp TestHelper.cpp Stats.cpp)
add_executable(checkRandomStateTest checkRandomStateTest.cpp TestHelper.cpp Stats.cpp)
target_link_libraries(testeth ${Boost_UNIT_TEST_FRAMEWORK_LIBRARIES})
target_link_libraries(testeth ${CURL_LIBRARIES})

83
test/Stats.cpp
View File

@ -0,0 +1,83 @@
/*
This file is part of cpp-ethereum.
cpp-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
cpp-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
*/
#include "Stats.h"
#include <iterator>
#include <numeric>
namespace dev
{
namespace test
{
Stats& Stats::get()
{
static Stats instance;
return instance;
}
void Stats::testStarted(std::string const& _name)
{
m_currentTest = _name;
m_tp = clock::now();
}
void Stats::testFinished()
{
m_stats[clock::now() - m_tp] = std::move(m_currentTest);
}
std::ostream& operator<<(std::ostream& out, Stats::clock::duration const& d)
{
return out << std::setw(10) << std::right << std::chrono::duration_cast<std::chrono::microseconds>(d).count() << " us";
}
Stats::~Stats()
{
if (m_stats.empty())
return;
auto& out = std::cout;
auto itr = m_stats.begin();
auto min = *itr;
auto max = *m_stats.rbegin();
std::advance(itr, m_stats.size() / 2);
auto med = *itr;
auto tot = std::accumulate(m_stats.begin(), m_stats.end(), clock::duration{}, [](clock::duration const& a, stats_t::value_type const& v)
{
return a + v.first;
});
out << "\nSTATS:\n\n" << std::setfill(' ');
if (Options::get().statsFull)
{
for (auto&& s: m_stats)
out << " " << std::setw(40) << std::left << s.second.substr(0, 40) << s.first << " \n";
out << "\n";
}
out << " tot: " << tot << "\n"
<< " avg: " << (tot / m_stats.size()) << "\n\n"
<< " min: " << min.first << " (" << min.second << ")\n"
<< " med: " << med.first << " (" << med.second << ")\n"
<< " max: " << max.first << " (" << max.second << ")\n";
}
}
}

50
test/Stats.h
View File

@ -0,0 +1,50 @@
/*
This file is part of cpp-ethereum.
cpp-ethereum is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
cpp-ethereum is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with cpp-ethereum. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <chrono>
#include <map>
#include "TestHelper.h"
namespace dev
{
namespace test
{
class Stats: public Listener
{
public:
using clock = std::chrono::high_resolution_clock;
using stats_t = std::map<clock::duration, std::string>;
static Stats& get();
~Stats();
void testStarted(std::string const& _name) override;
void testFinished() override;
private:
clock::time_point m_tp;
std::string m_currentTest;
stats_t m_stats;
};
}
}

28
test/TestHelper.cpp
View File

@ -551,6 +551,10 @@ Options::Options()
vmtrace = true;
else if (arg == "--filltests")
fillTests = true;
else if (arg == "--stats")
stats = true;
else if (arg == "--stats=full")
stats = statsFull = true;
else if (arg == "--performance")
performance = true;
else if (arg == "--quadratic")
@ -578,6 +582,7 @@ Options const& Options::get()
return instance;
}
LastHashes lastHashes(u256 _currentBlockNumber)
{
LastHashes ret;
@ -586,4 +591,27 @@ LastHashes lastHashes(u256 _currentBlockNumber)
return ret;
}
namespace
{
Listener* g_listener;
}
void Listener::registerListener(Listener& _listener)
{
g_listener = &_listener;
}
void Listener::notifyTestStarted(std::string const& _name)
{
if (g_listener)
g_listener->testStarted(_name);
}
void Listener::notifyTestFinished()
{
if (g_listener)
g_listener->testFinished();
}
} } // namespaces

30
test/TestHelper.h
View File

@ -162,8 +162,9 @@ class Options
public:
bool jit = false; ///< Use JIT
bool vmtrace = false; ///< Create EVM execution tracer // TODO: Link with log verbosity?
bool showTimes = false; ///< Print test groups execution times
bool fillTests = false; ///< Create JSON test files from execution results
bool stats = false; ///< Execution time stats
bool statsFull = false; ///< Output full stats - execution times for every test
/// Test selection
/// @{
@ -183,5 +184,32 @@ private:
Options(Options const&) = delete;
};
/// Allows observing test execution process.
/// This class also provides methods for registering and notifying the listener
class Listener
{
public:
virtual ~Listener() = default;
virtual void testStarted(std::string const& _name) = 0;
virtual void testFinished() = 0;
static void registerListener(Listener& _listener);
static void notifyTestStarted(std::string const& _name);
static void notifyTestFinished();
/// Test started/finished notification RAII helper
class ExecTimeGuard
{
public:
ExecTimeGuard(std::string const& _testName) { notifyTestStarted(_testName); }
~ExecTimeGuard() { notifyTestFinished(); }
ExecTimeGuard(ExecTimeGuard const&) = delete;
ExecTimeGuard& operator=(ExecTimeGuard) = delete;
};
};
}
}

21
test/state.cpp
View File

@ -31,6 +31,7 @@
#include <libethereum/Defaults.h>
#include <libevm/VM.h>
#include "TestHelper.h"
#include "Stats.h"
using namespace std;
using namespace json_spirit;
@ -41,7 +42,8 @@ namespace dev { namespace test {
void doStateTests(json_spirit::mValue& v, bool _fillin)
{
Options::get(); // process command line options
if (Options::get().stats)
Listener::registerListener(Stats::get());
for (auto& i: v.get_obj())
{
@ -60,6 +62,7 @@ void doStateTests(json_spirit::mValue& v, bool _fillin)
try
{
Listener::ExecTimeGuard guard{i.first};
theState.execute(lastHashes(importer.m_environment.currentBlock.number), tx, &output);
}
catch (Exception const& _e)
@ -178,29 +181,13 @@ BOOST_AUTO_TEST_CASE(stBlockHashTest)
BOOST_AUTO_TEST_CASE(stQuadraticComplexityTest)
{
if (test::Options::get().quadratic)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("stQuadraticComplexityTest", "/StateTests", dev::test::doStateTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(stMemoryStressTest)
{
if (test::Options::get().memory)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("stMemoryStressTest", "/StateTests", dev::test::doStateTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(stSolidityTest)

39
test/vm.cpp
View File

@ -20,13 +20,12 @@
* vm test functions.
*/
#include <chrono>
#include <boost/filesystem.hpp>
#include <libethereum/Executive.h>
#include <libevm/VMFactory.h>
#include "vm.h"
#include "Stats.h"
using namespace std;
using namespace json_spirit;
@ -312,7 +311,8 @@ namespace dev { namespace test {
void doVMTests(json_spirit::mValue& v, bool _fillin)
{
Options::get(); // process command line options // TODO: We need to control the main() function
if (Options::get().stats)
Listener::registerListener(Stats::get());
for (auto& i: v.get_obj())
{
@ -340,12 +340,16 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
bytes output;
u256 gas;
bool vmExceptionOccured = false;
auto startTime = std::chrono::high_resolution_clock::now();
try
{
auto vm = eth::VMFactory::create(fev.gas);
auto vmtrace = Options::get().vmtrace ? fev.simpleTrace() : OnOpFunc{};
output = vm->go(fev, vmtrace).toBytes();
auto outputRef = bytesConstRef{};
{
Listener::ExecTimeGuard guard{i.first};
outputRef = vm->go(fev, vmtrace);
}
output = outputRef.toBytes();
gas = vm->gas();
}
catch (VMException const&)
@ -364,15 +368,6 @@ void doVMTests(json_spirit::mValue& v, bool _fillin)
BOOST_ERROR("Failed VM Test with Exception: " << _e.what());
}
auto endTime = std::chrono::high_resolution_clock::now();
if (Options::get().showTimes)
{
auto testDuration = endTime - startTime;
cnote << "Execution time: "
<< std::chrono::duration_cast<std::chrono::milliseconds>(testDuration).count()
<< " ms";
}
// delete null entries in storage for the sake of comparison
for (auto &a: fev.addresses)
@ -513,29 +508,13 @@ BOOST_AUTO_TEST_CASE(vmSystemOperationsTest)
BOOST_AUTO_TEST_CASE(vmPerformanceTest)
{
if (test::Options::get().performance)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("vmPerformanceTest", "/VMTests", dev::test::doVMTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(vmInputLimitsTest1)
{
if (test::Options::get().inputLimits)
{
auto start = chrono::steady_clock::now();
dev::test::executeTests("vmInputLimits1", "/VMTests", dev::test::doVMTests);
auto end = chrono::steady_clock::now();
auto duration(chrono::duration_cast<chrono::milliseconds>(end - start));
cnote << "test duration: " << duration.count() << " milliseconds.\n";
}
}
BOOST_AUTO_TEST_CASE(vmInputLimitsTest2)

Write Preview
Loading…
Cancel
Save